diff options
| author | Sebastian Huber <sebastian.huber@embedded-brains.de> | 2018-08-21 13:47:02 +0200 |
|---|---|---|
| committer | Sebastian Huber <sebastian.huber@embedded-brains.de> | 2018-09-21 10:29:41 +0200 |
| commit | bcdce02d9bc8150e1d191ed5ca9da45b7604964a (patch) | |
| tree | 3b2faf509db7672ee1fc98857736470be97e7ed8 /freebsd | |
| parent | Update to FreeBSD head 2018-04-01 (diff) | |
| download | rtems-libbsd-bcdce02d9bc8150e1d191ed5ca9da45b7604964a.tar.bz2 | |
Update to FreeBSD head 2018-06-01
Git mirror commit fb63610a69b0eb7f69a201ba05c4c1a7a2739cf9.
Update #3472.
Diffstat (limited to 'freebsd')
327 files changed, 24188 insertions, 11780 deletions
diff --git a/freebsd/contrib/libpcap/bpf_image.c b/freebsd/contrib/libpcap/bpf_image.c index 5ab1b2a8..2bf71c6d 100644 --- a/freebsd/contrib/libpcap/bpf_image.c +++ b/freebsd/contrib/libpcap/bpf_image.c @@ -22,22 +22,10 @@ */ #ifdef HAVE_CONFIG_H -#include "config.h" +#include <config.h> #endif -#ifdef _WIN32 -#include <pcap-stdinc.h> -#else /* _WIN32 */ -#if HAVE_INTTYPES_H -#include <inttypes.h> -#elif HAVE_STDINT_H -#include <stdint.h> -#endif -#ifdef HAVE_SYS_BITYPES_H -#include <sys/bitypes.h> -#endif -#include <sys/types.h> -#endif /* _WIN32 */ +#include <pcap-types.h> #include <stdio.h> #include <string.h> @@ -49,266 +37,290 @@ #endif char * -bpf_image(p, n) - const struct bpf_insn *p; - int n; +bpf_image(const struct bpf_insn *p, int n) { - int v; - const char *fmt, *op; + const char *op; static char image[256]; - char operand[64]; + char operand_buf[64]; + const char *operand; - v = p->k; switch (p->code) { default: op = "unimp"; - fmt = "0x%x"; - v = p->code; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "0x%x", p->code); + operand = operand_buf; break; case BPF_RET|BPF_K: op = "ret"; - fmt = "#%d"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "#%d", p->k); + operand = operand_buf; break; case BPF_RET|BPF_A: op = "ret"; - fmt = ""; + operand = ""; break; case BPF_LD|BPF_W|BPF_ABS: op = "ld"; - fmt = "[%d]"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "[%d]", p->k); + operand = operand_buf; break; case BPF_LD|BPF_H|BPF_ABS: op = "ldh"; - fmt = "[%d]"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "[%d]", p->k); + operand = operand_buf; break; case BPF_LD|BPF_B|BPF_ABS: op = "ldb"; - fmt = "[%d]"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "[%d]", p->k); + operand = operand_buf; break; case BPF_LD|BPF_W|BPF_LEN: op = "ld"; - fmt = "#pktlen"; + operand = "#pktlen"; break; case BPF_LD|BPF_W|BPF_IND: op = "ld"; - fmt = "[x + %d]"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "[x + %d]", p->k); + operand = operand_buf; break; case BPF_LD|BPF_H|BPF_IND: op = "ldh"; - fmt = "[x + %d]"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "[x + %d]", p->k); + operand = operand_buf; break; case BPF_LD|BPF_B|BPF_IND: op = "ldb"; - fmt = "[x + %d]"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "[x + %d]", p->k); + operand = operand_buf; break; case BPF_LD|BPF_IMM: op = "ld"; - fmt = "#0x%x"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "#0x%x", p->k); + operand = operand_buf; break; case BPF_LDX|BPF_IMM: op = "ldx"; - fmt = "#0x%x"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "#0x%x", p->k); + operand = operand_buf; break; case BPF_LDX|BPF_MSH|BPF_B: op = "ldxb"; - fmt = "4*([%d]&0xf)"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "4*([%d]&0xf)", p->k); + operand = operand_buf; break; case BPF_LD|BPF_MEM: op = "ld"; - fmt = "M[%d]"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "M[%d]", p->k); + operand = operand_buf; break; case BPF_LDX|BPF_MEM: op = "ldx"; - fmt = "M[%d]"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "M[%d]", p->k); + operand = operand_buf; break; case BPF_ST: op = "st"; - fmt = "M[%d]"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "M[%d]", p->k); + operand = operand_buf; break; case BPF_STX: op = "stx"; - fmt = "M[%d]"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "M[%d]", p->k); + operand = operand_buf; break; case BPF_JMP|BPF_JA: op = "ja"; - fmt = "%d"; - v = n + 1 + p->k; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "%d", n + 1 + p->k); + operand = operand_buf; break; case BPF_JMP|BPF_JGT|BPF_K: op = "jgt"; - fmt = "#0x%x"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "#0x%x", p->k); + operand = operand_buf; break; case BPF_JMP|BPF_JGE|BPF_K: op = "jge"; - fmt = "#0x%x"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "#0x%x", p->k); + operand = operand_buf; break; case BPF_JMP|BPF_JEQ|BPF_K: op = "jeq"; - fmt = "#0x%x"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "#0x%x", p->k); + operand = operand_buf; break; case BPF_JMP|BPF_JSET|BPF_K: op = "jset"; - fmt = "#0x%x"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "#0x%x", p->k); + operand = operand_buf; break; case BPF_JMP|BPF_JGT|BPF_X: op = "jgt"; - fmt = "x"; + operand = "x"; break; case BPF_JMP|BPF_JGE|BPF_X: op = "jge"; - fmt = "x"; + operand = "x"; break; case BPF_JMP|BPF_JEQ|BPF_X: op = "jeq"; - fmt = "x"; + operand = "x"; break; case BPF_JMP|BPF_JSET|BPF_X: op = "jset"; - fmt = "x"; + operand = "x"; break; case BPF_ALU|BPF_ADD|BPF_X: op = "add"; - fmt = "x"; + operand = "x"; break; case BPF_ALU|BPF_SUB|BPF_X: op = "sub"; - fmt = "x"; + operand = "x"; break; case BPF_ALU|BPF_MUL|BPF_X: op = "mul"; - fmt = "x"; + operand = "x"; break; case BPF_ALU|BPF_DIV|BPF_X: op = "div"; - fmt = "x"; + operand = "x"; break; case BPF_ALU|BPF_MOD|BPF_X: op = "mod"; - fmt = "x"; + operand = "x"; break; case BPF_ALU|BPF_AND|BPF_X: op = "and"; - fmt = "x"; + operand = "x"; break; case BPF_ALU|BPF_OR|BPF_X: op = "or"; - fmt = "x"; + operand = "x"; break; case BPF_ALU|BPF_XOR|BPF_X: op = "xor"; - fmt = "x"; + operand = "x"; break; case BPF_ALU|BPF_LSH|BPF_X: op = "lsh"; - fmt = "x"; + operand = "x"; break; case BPF_ALU|BPF_RSH|BPF_X: op = "rsh"; - fmt = "x"; + operand = "x"; break; case BPF_ALU|BPF_ADD|BPF_K: op = "add"; - fmt = "#%d"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "#%d", p->k); + operand = operand_buf; break; case BPF_ALU|BPF_SUB|BPF_K: op = "sub"; - fmt = "#%d"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "#%d", p->k); + operand = operand_buf; break; case BPF_ALU|BPF_MUL|BPF_K: op = "mul"; - fmt = "#%d"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "#%d", p->k); + operand = operand_buf; break; case BPF_ALU|BPF_DIV|BPF_K: op = "div"; - fmt = "#%d"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "#%d", p->k); + operand = operand_buf; break; case BPF_ALU|BPF_MOD|BPF_K: op = "mod"; - fmt = "#%d"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "#%d", p->k); + operand = operand_buf; break; case BPF_ALU|BPF_AND|BPF_K: op = "and"; - fmt = "#0x%x"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "#0x%x", p->k); + operand = operand_buf; break; case BPF_ALU|BPF_OR|BPF_K: op = "or"; - fmt = "#0x%x"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "#0x%x", p->k); + operand = operand_buf; break; case BPF_ALU|BPF_XOR|BPF_K: op = "xor"; - fmt = "#0x%x"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "#0x%x", p->k); + operand = operand_buf; break; case BPF_ALU|BPF_LSH|BPF_K: op = "lsh"; - fmt = "#%d"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "#%d", p->k); + operand = operand_buf; break; case BPF_ALU|BPF_RSH|BPF_K: op = "rsh"; - fmt = "#%d"; + (void)pcap_snprintf(operand_buf, sizeof operand_buf, "#%d", p->k); + operand = operand_buf; break; case BPF_ALU|BPF_NEG: op = "neg"; - fmt = ""; + operand = ""; break; case BPF_MISC|BPF_TAX: op = "tax"; - fmt = ""; + operand = ""; break; case BPF_MISC|BPF_TXA: op = "txa"; - fmt = ""; + operand = ""; break; } - (void)pcap_snprintf(operand, sizeof operand, fmt, v); if (BPF_CLASS(p->code) == BPF_JMP && BPF_OP(p->code) != BPF_JA) { (void)pcap_snprintf(image, sizeof image, "(%03d) %-8s %-16s jt %d\tjf %d", diff --git a/freebsd/contrib/libpcap/diag-control.h b/freebsd/contrib/libpcap/diag-control.h new file mode 100644 index 00000000..12770361 --- /dev/null +++ b/freebsd/contrib/libpcap/diag-control.h @@ -0,0 +1,215 @@ +/* -*- Mode: c; tab-width: 8; indent-tabs-mode: 1; c-basic-offset: 8; -*- */ +/* + * Copyright (c) 1993, 1994, 1995, 1996, 1997 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the Computer Systems + * Engineering Group at Lawrence Berkeley Laboratory. + * 4. Neither the name of the University nor of the Laboratory may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _diag_control_h +#define _diag_control_h + +#include "pcap/compiler-tests.h" + +#ifndef _MSC_VER + /* + * Clang and GCC both support this way of putting pragmas into #defines. + * We don't use it unless we have a compiler that supports it; the + * warning-suppressing pragmas differ between Clang and GCC, so we test + * for both of those separately. + */ + #define PCAP_DO_PRAGMA(x) _Pragma (#x) +#endif + +/* + * Suppress Flex warnings. + */ +#if defined(_MSC_VER) + /* + * This is Microsoft Visual Studio; we can use __pragma(warning(disable:XXXX)) + * and __pragma(warning(push/pop)). + * + * Suppress signed-vs-unsigned comparison, narrowing, and unreachable + * code warnings. + */ + #define DIAG_OFF_FLEX \ + __pragma(warning(push)) \ + __pragma(warning(disable:4127)) \ + __pragma(warning(disable:4242)) \ + __pragma(warning(disable:4244)) \ + __pragma(warning(disable:4702)) + #define DIAG_ON_FLEX __pragma(warning(pop)) +#elif PCAP_IS_AT_LEAST_CLANG_VERSION(2,8) + /* + * This is Clang 2.8 or later; we can use "clang diagnostic + * ignored -Wxxx" and "clang diagnostic push/pop". + * + * Suppress -Wdocumentation warnings; GCC doesn't support -Wdocumentation, + * at least according to the GCC 7.3 documentation. Apparently, Flex + * generates code that upsets at least some versions of Clang's + * -Wdocumentation. + */ + #define DIAG_OFF_FLEX \ + PCAP_DO_PRAGMA(clang diagnostic push) \ + PCAP_DO_PRAGMA(clang diagnostic ignored "-Wsign-compare") \ + PCAP_DO_PRAGMA(clang diagnostic ignored "-Wdocumentation") \ + PCAP_DO_PRAGMA(clang diagnostic ignored "-Wmissing-noreturn") \ + PCAP_DO_PRAGMA(clang diagnostic ignored "-Wunused-parameter") \ + PCAP_DO_PRAGMA(clang diagnostic ignored "-Wunreachable-code") + #define DIAG_ON_FLEX \ + PCAP_DO_PRAGMA(clang diagnostic pop) +#elif PCAP_IS_AT_LEAST_GNUC_VERSION(4,6) + /* + * This is GCC 4.6 or later, or a compiler claiming to be that. + * We can use "GCC diagnostic ignored -Wxxx" (introduced in 4.2) + * and "GCC diagnostic push/pop" (introduced in 4.6). + */ + #define DIAG_OFF_FLEX \ + PCAP_DO_PRAGMA(GCC diagnostic push) \ + PCAP_DO_PRAGMA(GCC diagnostic ignored "-Wsign-compare") \ + PCAP_DO_PRAGMA(GCC diagnostic ignored "-Wunused-parameter") \ + PCAP_DO_PRAGMA(GCC diagnostic ignored "-Wunreachable-code") + #define DIAG_ON_FLEX \ + PCAP_DO_PRAGMA(GCC diagnostic pop) +#else + /* + * Neither Visual Studio, nor Clang 2.8 or later, nor GCC 4.6 or later + * or a compiler claiming to be that; there's nothing we know of that + * we can do. + */ + #define DIAG_OFF_FLEX + #define DIAG_ON_FLEX +#endif + +#ifdef YYBYACC + /* + * Berkeley YACC. + * + * It generates a global declaration of yylval, or the appropriately + * prefixed version of yylval, in grammar.h, *even though it's been + * told to generate a pure parser, meaning it doesn't have any global + * variables*. Bison doesn't do this. + * + * That causes a warning due to the local declaration in the parser + * shadowing the global declaration. + * + * So, if the compiler warns about that, we turn off -Wshadow warnings. + */ + #if defined(_MSC_VER) + /* + * This is Microsoft Visual Studio; we can use + * __pragma(warning(disable:XXXX)) and __pragma(warning(push/pop)). + * + * Suppress unreachable code warnings. + */ + #define DIAG_OFF_BISON_BYACC \ + __pragma(warning(push)) \ + __pragma(warning(disable:4702)) + #define DIAG_ON_BISON_BYACC __pragma(warning(pop)) + #elif PCAP_IS_AT_LEAST_CLANG_VERSION(2,8) + /* + * This is Clang 2.8 or later; we can use "clang diagnostic + * ignored -Wxxx" and "clang diagnostic push/pop". + */ + #define DIAG_OFF_BISON_BYACC \ + PCAP_DO_PRAGMA(clang diagnostic push) \ + PCAP_DO_PRAGMA(clang diagnostic ignored "-Wshadow") \ + PCAP_DO_PRAGMA(clang diagnostic ignored "-Wunreachable-code") + #define DIAG_ON_BISON_BYACC \ + PCAP_DO_PRAGMA(clang diagnostic pop) + #elif PCAP_IS_AT_LEAST_GNUC_VERSION(4,6) + /* + * This is GCC 4.6 or later, or a compiler claiming to be that. + * We can use "GCC diagnostic ignored -Wxxx" (introduced in 4.2) + * and "GCC diagnostic push/pop" (introduced in 4.6). + */ + #define DIAG_OFF_BISON_BYACC \ + PCAP_DO_PRAGMA(GCC diagnostic push) \ + PCAP_DO_PRAGMA(GCC diagnostic ignored "-Wshadow") \ + PCAP_DO_PRAGMA(GCC diagnostic ignored "-Wunreachable-code") + #define DIAG_ON_BISON_BYACC \ + PCAP_DO_PRAGMA(GCC diagnostic pop) + #else + /* + * Neither Clang 2.8 or later nor GCC 4.6 or later or a compiler + * claiming to be that; there's nothing we know of that we can do. + */ + #define DIAG_OFF_BISON_BYACC + #define DIAG_ON_BISON_BYACC + #endif +#else + /* + * Bison. + */ + #if defined(_MSC_VER) + /* + * This is Microsoft Visual Studio; we can use + * __pragma(warning(disable:XXXX)) and __pragma(warning(push/pop)). + * + * Suppress some /Wall warnings. + */ + #define DIAG_OFF_BISON_BYACC \ + __pragma(warning(push)) \ + __pragma(warning(disable:4127)) \ + __pragma(warning(disable:4242)) \ + __pragma(warning(disable:4244)) \ + __pragma(warning(disable:4702)) + #define DIAG_ON_BISON_BYACC __pragma(warning(pop)) + #elif PCAP_IS_AT_LEAST_CLANG_VERSION(2,8) + /* + * This is Clang 2.8 or later; we can use "clang diagnostic + * ignored -Wxxx" and "clang diagnostic push/pop". + */ + #define DIAG_OFF_BISON_BYACC \ + PCAP_DO_PRAGMA(clang diagnostic push) \ + PCAP_DO_PRAGMA(clang diagnostic ignored "-Wunreachable-code") + #define DIAG_ON_BISON_BYACC \ + PCAP_DO_PRAGMA(clang diagnostic pop) + #elif PCAP_IS_AT_LEAST_GNUC_VERSION(4,6) + /* + * This is GCC 4.6 or later, or a compiler claiming to be that. + * We can use "GCC diagnostic ignored -Wxxx" (introduced in 4.2) + * and "GCC diagnostic push/pop" (introduced in 4.6). + */ + #define DIAG_OFF_BISON_BYACC \ + PCAP_DO_PRAGMA(GCC diagnostic push) \ + PCAP_DO_PRAGMA(GCC diagnostic ignored "-Wunreachable-code") + #define DIAG_ON_BISON_BYACC \ + PCAP_DO_PRAGMA(GCC diagnostic pop) + #else + /* + * Neither Clang 2.8 or later nor GCC 4.6 or later or a compiler + * claiming to be that; there's nothing we know of that we can do. + */ + #define DIAG_OFF_BISON_BYACC + #define DIAG_ON_BISON_BYACC + #endif +#endif + +#endif /* _diag_control_h */ diff --git a/freebsd/contrib/libpcap/etherent.c b/freebsd/contrib/libpcap/etherent.c index 0291acc1..0522ac9b 100644 --- a/freebsd/contrib/libpcap/etherent.c +++ b/freebsd/contrib/libpcap/etherent.c @@ -22,22 +22,10 @@ */ #ifdef HAVE_CONFIG_H -#include "config.h" +#include <config.h> #endif -#ifdef _WIN32 -#include <pcap-stdinc.h> -#else /* _WIN32 */ -#if HAVE_INTTYPES_H -#include <inttypes.h> -#elif HAVE_STDINT_H -#include <stdint.h> -#endif -#ifdef HAVE_SYS_BITYPES_H -#include <sys/bitypes.h> -#endif -#include <sys/types.h> -#endif /* _WIN32 */ +#include <pcap-types.h> #include <ctype.h> #include <memory.h> @@ -52,26 +40,23 @@ #include "os-proto.h" #endif -static inline int xdtoi(int); static inline int skip_space(FILE *); static inline int skip_line(FILE *); /* Hex digit to integer. */ -static inline int -xdtoi(c) - register int c; +static inline u_char +xdtoi(u_char c) { if (isdigit(c)) - return c - '0'; + return (u_char)(c - '0'); else if (islower(c)) - return c - 'a' + 10; + return (u_char)(c - 'a' + 10); else - return c - 'A' + 10; + return (u_char)(c - 'A' + 10); } static inline int -skip_space(f) - FILE *f; +skip_space(FILE *f) { int c; @@ -83,8 +68,7 @@ skip_space(f) } static inline int -skip_line(f) - FILE *f; +skip_line(FILE *f) { int c; @@ -98,47 +82,61 @@ skip_line(f) struct pcap_etherent * pcap_next_etherent(FILE *fp) { - register int c, d, i; + register int c, i; + u_char d; char *bp; + size_t namesize; static struct pcap_etherent e; memset((char *)&e, 0, sizeof(e)); - do { + for (;;) { /* Find addr */ c = skip_space(fp); + if (c == EOF) + return (NULL); if (c == '\n') continue; /* If this is a comment, or first thing on line - cannot be etehrnet address, skip the line. */ + cannot be Ethernet address, skip the line. */ if (!isxdigit(c)) { c = skip_line(fp); + if (c == EOF) + return (NULL); continue; } /* must be the start of an address */ for (i = 0; i < 6; i += 1) { - d = xdtoi(c); + d = xdtoi((u_char)c); c = getc(fp); + if (c == EOF) + return (NULL); if (isxdigit(c)) { d <<= 4; - d |= xdtoi(c); + d |= xdtoi((u_char)c); c = getc(fp); + if (c == EOF) + return (NULL); } e.addr[i] = d; if (c != ':') break; c = getc(fp); + if (c == EOF) + return (NULL); } - if (c == EOF) - break; /* Must be whitespace */ if (!isspace(c)) { c = skip_line(fp); + if (c == EOF) + return (NULL); continue; } c = skip_space(fp); + if (c == EOF) + return (NULL); /* hit end of line... */ if (c == '\n') @@ -146,17 +144,21 @@ pcap_next_etherent(FILE *fp) if (c == '#') { c = skip_line(fp); + if (c == EOF) + return (NULL); continue; } /* pick up name */ bp = e.name; - /* Use 'd' to prevent buffer overflow. */ - d = sizeof(e.name) - 1; + /* Use 'namesize' to prevent buffer overflow. */ + namesize = sizeof(e.name) - 1; do { - *bp++ = c; + *bp++ = (u_char)c; c = getc(fp); - } while (!isspace(c) && c != EOF && --d > 0); + if (c == EOF) + return (NULL); + } while (!isspace(c) && --namesize != 0); *bp = '\0'; /* Eat trailing junk */ @@ -164,8 +166,5 @@ pcap_next_etherent(FILE *fp) (void)skip_line(fp); return &e; - - } while (c != EOF); - - return (NULL); + } } diff --git a/freebsd/contrib/libpcap/extract.h b/freebsd/contrib/libpcap/extract.h index face5b7e..aa3ff991 100644 --- a/freebsd/contrib/libpcap/extract.h +++ b/freebsd/contrib/libpcap/extract.h @@ -23,6 +23,9 @@ #include <arpa/inet.h> #endif +#include <pcap/pcap-inttypes.h> +#include <pcap/compiler-tests.h> + /* * Macros to extract possibly-unaligned big-endian integral values. */ @@ -30,15 +33,16 @@ /* * The processor doesn't natively handle unaligned loads. */ -#if defined(__GNUC__) && defined(HAVE___ATTRIBUTE__) && \ +#if PCAP_IS_AT_LEAST_GNUC_VERSION(2,0) && \ (defined(__alpha) || defined(__alpha__) || \ defined(__mips) || defined(__mips__)) - /* - * This is a GCC-compatible compiler and we have __attribute__, which - * we assume that mean we have __attribute__((packed)), and this is - * MIPS or Alpha, which has instructions that can help when doing - * unaligned loads. + * This is MIPS or Alpha, which don't natively handle unaligned loads, + * but which have instructions that can help when doing unaligned + * loads, and this is GCC 2.0 or later or a compiler that claims to + * be GCC 2.0 or later, which we assume that mean we have + * __attribute__((packed)), which we can use to convince the compiler + * to generate those instructions. * * Declare packed structures containing a uint16_t and a uint32_t, * cast the pointer to point to one of those, and fetch through it; diff --git a/freebsd/contrib/libpcap/fad-getad.c b/freebsd/contrib/libpcap/fad-getad.c index cd7ac3e6..620f0d52 100644 --- a/freebsd/contrib/libpcap/fad-getad.c +++ b/freebsd/contrib/libpcap/fad-getad.c @@ -35,7 +35,7 @@ */ #ifdef HAVE_CONFIG_H -#include "config.h" +#include <config.h> #endif #include <sys/types.h> @@ -104,10 +104,10 @@ * all those systems we have "struct sockaddr_storage". */ #ifndef SA_LEN -#ifdef HAVE_SOCKADDR_SA_LEN +#ifdef HAVE_STRUCT_SOCKADDR_SA_LEN #define SA_LEN(addr) ((addr)->sa_len) -#else /* HAVE_SOCKADDR_SA_LEN */ -#ifdef HAVE_SOCKADDR_STORAGE +#else /* HAVE_STRUCT_SOCKADDR_SA_LEN */ +#ifdef HAVE_STRUCT_SOCKADDR_STORAGE static size_t get_sa_len(struct sockaddr *addr) { @@ -133,10 +133,10 @@ get_sa_len(struct sockaddr *addr) } } #define SA_LEN(addr) (get_sa_len(addr)) -#else /* HAVE_SOCKADDR_STORAGE */ +#else /* HAVE_STRUCT_SOCKADDR_STORAGE */ #define SA_LEN(addr) (sizeof (struct sockaddr)) -#endif /* HAVE_SOCKADDR_STORAGE */ -#endif /* HAVE_SOCKADDR_SA_LEN */ +#endif /* HAVE_STRUCT_SOCKADDR_STORAGE */ +#endif /* HAVE_STRUCT_SOCKADDR_SA_LEN */ #endif /* SA_LEN */ /* @@ -146,10 +146,9 @@ get_sa_len(struct sockaddr *addr) * could be opened. */ int -pcap_findalldevs_interfaces(pcap_if_t **alldevsp, char *errbuf, - int (*check_usable)(const char *)) +pcap_findalldevs_interfaces(pcap_if_list_t *devlistp, char *errbuf, + int (*check_usable)(const char *), get_if_flags_func get_flags_func) { - pcap_if_t *devlist = NULL; struct ifaddrs *ifap, *ifa; struct sockaddr *addr, *netmask, *broadaddr, *dstaddr; size_t addr_size, broadaddr_size, dstaddr_size; @@ -171,8 +170,8 @@ pcap_findalldevs_interfaces(pcap_if_t **alldevsp, char *errbuf, * those. */ if (getifaddrs(&ifap) != 0) { - (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "getifaddrs: %s", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "getifaddrs"); return (-1); } for (ifa = ifap; ifa != NULL; ifa = ifa->ifa_next) { @@ -235,7 +234,7 @@ pcap_findalldevs_interfaces(pcap_if_t **alldevsp, char *errbuf, /* * Note that, on some platforms, ifa_broadaddr and * ifa_dstaddr could be the same field (true on at - * least some versions of *BSD and OS X), so we + * least some versions of *BSD and macOS), so we * can't just check whether the broadcast address * is null and add it if so and check whether the * destination address is null and add it if so. @@ -267,8 +266,8 @@ pcap_findalldevs_interfaces(pcap_if_t **alldevsp, char *errbuf, /* * Add information for this address to the list. */ - if (add_addr_to_iflist(&devlist, ifa->ifa_name, - if_flags_to_pcap_flags(ifa->ifa_name, ifa->ifa_flags), + if (add_addr_to_if(devlistp, ifa->ifa_name, ifa->ifa_flags, + get_flags_func, addr, addr_size, netmask, addr_size, broadaddr, broadaddr_size, dstaddr, dstaddr_size, errbuf) < 0) { @@ -279,16 +278,5 @@ pcap_findalldevs_interfaces(pcap_if_t **alldevsp, char *errbuf, freeifaddrs(ifap); - if (ret == -1) { - /* - * We had an error; free the list we've been constructing. - */ - if (devlist != NULL) { - pcap_freealldevs(devlist); - devlist = NULL; - } - } - - *alldevsp = devlist; return (ret); } diff --git a/freebsd/contrib/libpcap/fad-helpers.c b/freebsd/contrib/libpcap/fad-helpers.c deleted file mode 100644 index 0b98b484..00000000 --- a/freebsd/contrib/libpcap/fad-helpers.c +++ /dev/null @@ -1,890 +0,0 @@ -#include <machine/rtems-bsd-user-space.h> - -/* -*- Mode: c; tab-width: 8; indent-tabs-mode: 1; c-basic-offset: 8; -*- */ -/* - * Copyright (c) 1994, 1995, 1996, 1997, 1998 - * The Regents of the University of California. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the Computer Systems - * Engineering Group at Lawrence Berkeley Laboratory. - * 4. Neither the name of the University nor of the Laboratory may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#ifdef _WIN32 -#include <pcap-stdinc.h> -#else /* _WIN32 */ - -#include <sys/param.h> -#ifndef MSDOS -#include <sys/file.h> -#endif -#include <sys/ioctl.h> -#include <sys/socket.h> -#ifdef HAVE_SYS_SOCKIO_H -#include <sys/sockio.h> -#endif - -struct mbuf; /* Squelch compiler warnings on some platforms for */ -struct rtentry; /* declarations in <net/if.h> */ -#include <net/if.h> -#include <netinet/in.h> -#endif /* _WIN32 */ - -#include <ctype.h> -#include <errno.h> -#include <memory.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#if !defined(_WIN32) && !defined(__BORLANDC__) -#include <unistd.h> -#endif /* !_WIN32 && !__BORLANDC__ */ -#ifdef HAVE_LIMITS_H -#include <limits.h> -#else -#define INT_MAX 2147483647 -#endif - -#include "pcap-int.h" - -#ifdef HAVE_OS_PROTO_H -#include "os-proto.h" -#endif - -#ifndef _WIN32 -/* Not all systems have IFF_LOOPBACK */ -#ifdef IFF_LOOPBACK -#define ISLOOPBACK(name, flags) ((flags) & IFF_LOOPBACK) -#else -#define ISLOOPBACK(name, flags) ((name)[0] == 'l' && (name)[1] == 'o' && \ - (isdigit((unsigned char)((name)[2])) || (name)[2] == '\0')) -#endif - -#ifdef IFF_UP -#define ISUP(flags) ((flags) & IFF_UP) -#else -#define ISUP(flags) 0 -#endif - -#ifdef IFF_RUNNING -#define ISRUNNING(flags) ((flags) & IFF_RUNNING) -#else -#define ISRUNNING(flags) 0 -#endif - -/* - * Map UN*X-style interface flags to libpcap flags. - */ -bpf_u_int32 -if_flags_to_pcap_flags(const char *name _U_, u_int if_flags) -{ - bpf_u_int32 pcap_flags; - - pcap_flags = 0; - if (ISLOOPBACK(name, if_flags)) - pcap_flags |= PCAP_IF_LOOPBACK; - if (ISUP(if_flags)) - pcap_flags |= PCAP_IF_UP; - if (ISRUNNING(if_flags)) - pcap_flags |= PCAP_IF_RUNNING; - return (pcap_flags); -} -#endif - -static struct sockaddr * -dup_sockaddr(struct sockaddr *sa, size_t sa_length) -{ - struct sockaddr *newsa; - - if ((newsa = malloc(sa_length)) == NULL) - return (NULL); - return (memcpy(newsa, sa, sa_length)); -} - -/* - * Construct a "figure of merit" for an interface, for use when sorting - * the list of interfaces, in which interfaces that are up are superior - * to interfaces that aren't up, interfaces that are up and running are - * superior to interfaces that are up but not running, and non-loopback - * interfaces that are up and running are superior to loopback interfaces, - * and interfaces with the same flags have a figure of merit that's higher - * the lower the instance number. - * - * The goal is to try to put the interfaces most likely to be useful for - * capture at the beginning of the list. - * - * The figure of merit, which is lower the "better" the interface is, - * has the uppermost bit set if the interface isn't running, the bit - * below that set if the interface isn't up, the bit below that set - * if the interface is a loopback interface, and the interface index - * in the 29 bits below that. (Yes, we assume u_int is 32 bits.) - */ -static u_int -get_figure_of_merit(pcap_if_t *dev) -{ - const char *cp; - u_int n; - - if (strcmp(dev->name, "any") == 0) { - /* - * Give the "any" device an artificially high instance - * number, so it shows up after all other non-loopback - * interfaces. - */ - n = 0x1FFFFFFF; /* 29 all-1 bits */ - } else { - /* - * A number at the end of the device name string is - * assumed to be a unit number. - */ - cp = dev->name + strlen(dev->name) - 1; - while (cp-1 >= dev->name && *(cp-1) >= '0' && *(cp-1) <= '9') - cp--; - if (*cp >= '0' && *cp <= '9') - n = atoi(cp); - else - n = 0; - } - if (!(dev->flags & PCAP_IF_RUNNING)) - n |= 0x80000000; - if (!(dev->flags & PCAP_IF_UP)) - n |= 0x40000000; - if (dev->flags & PCAP_IF_LOOPBACK) - n |= 0x20000000; - return (n); -} - -/* - * Try to get a description for a given device. - * Returns a mallocated description if it could and NULL if it couldn't. - * - * XXX - on FreeBSDs that support it, should it get the sysctl named - * "dev.{adapter family name}.{adapter unit}.%desc" to get a description - * of the adapter? Note that "dev.an.0.%desc" is "Aironet PC4500/PC4800" - * with my Cisco 350 card, so the name isn't entirely descriptive. The - * "dev.an.0.%pnpinfo" has a better description, although one might argue - * that the problem is really a driver bug - if it can find out that it's - * a Cisco 340 or 350, rather than an old Aironet card, it should use - * that in the description. - * - * Do NetBSD, DragonflyBSD, or OpenBSD support this as well? FreeBSD - * and OpenBSD let you get a description, but it's not generated by the OS, - * it's set with another ioctl that ifconfig supports; we use that to get - * a description in FreeBSD and OpenBSD, but if there is no such - * description available, it still might be nice to get some description - * string based on the device type or something such as that. - * - * In OS X, the System Configuration framework can apparently return - * names in 10.4 and later. - * - * It also appears that freedesktop.org's HAL offers an "info.product" - * string, but the HAL specification says it "should not be used in any - * UI" and "subsystem/capability specific properties" should be used - * instead and, in any case, I think HAL is being deprecated in - * favor of other stuff such as DeviceKit. DeviceKit doesn't appear - * to have any obvious product information for devices, but maybe - * I haven't looked hard enough. - * - * Using the System Configuration framework, or HAL, or DeviceKit, or - * whatever, would require that libpcap applications be linked with - * the frameworks/libraries in question. That shouldn't be a problem - * for programs linking with the shared version of libpcap (unless - * you're running on AIX - which I think is the only UN*X that doesn't - * support linking a shared library with other libraries on which it - * depends, and having an executable linked only with the first shared - * library automatically pick up the other libraries when started - - * and using HAL or whatever). Programs linked with the static - * version of libpcap would have to use pcap-config with the --static - * flag in order to get the right linker flags in order to pick up - * the additional libraries/frameworks; those programs need that anyway - * for libpcap 1.1 and beyond on Linux, as, by default, it requires - * -lnl. - * - * Do any other UN*Xes, or desktop environments support getting a - * description? - */ -static char * -get_if_description(const char *name) -{ -#ifdef SIOCGIFDESCR - char *description = NULL; - int s; - struct ifreq ifrdesc; -#ifndef IFDESCRSIZE - size_t descrlen = 64; -#else - size_t descrlen = IFDESCRSIZE; -#endif /* IFDESCRSIZE */ - - /* - * Get the description for the interface. - */ - memset(&ifrdesc, 0, sizeof ifrdesc); - strlcpy(ifrdesc.ifr_name, name, sizeof ifrdesc.ifr_name); - s = socket(AF_INET, SOCK_DGRAM, 0); - if (s >= 0) { -#ifdef __FreeBSD__ - /* - * On FreeBSD, if the buffer isn't big enough for the - * description, the ioctl succeeds, but the description - * isn't copied, ifr_buffer.length is set to the description - * length, and ifr_buffer.buffer is set to NULL. - */ - for (;;) { - free(description); - if ((description = malloc(descrlen)) != NULL) { - ifrdesc.ifr_buffer.buffer = description; - ifrdesc.ifr_buffer.length = descrlen; - if (ioctl(s, SIOCGIFDESCR, &ifrdesc) == 0) { - if (ifrdesc.ifr_buffer.buffer == - description) - break; - else - descrlen = ifrdesc.ifr_buffer.length; - } else { - /* - * Failed to get interface description. - */ - free(description); - description = NULL; - break; - } - } else - break; - } -#else /* __FreeBSD__ */ - /* - * The only other OS that currently supports - * SIOCGIFDESCR is OpenBSD, and it has no way - * to get the description length - it's clamped - * to a maximum of IFDESCRSIZE. - */ - if ((description = malloc(descrlen)) != NULL) { - ifrdesc.ifr_data = (caddr_t)description; - if (ioctl(s, SIOCGIFDESCR, &ifrdesc) != 0) { - /* - * Failed to get interface description. - */ - free(description); - description = NULL; - } - } -#endif /* __FreeBSD__ */ - close(s); - if (description != NULL && strlen(description) == 0) { - /* - * Description is empty, so discard it. - */ - free(description); - description = NULL; - } - } - -#ifdef __FreeBSD__ - /* - * For FreeBSD, if we didn't get a description, and this is - * a device with a name of the form usbusN, label it as a USB - * bus. - */ - if (description == NULL) { - if (strncmp(name, "usbus", 5) == 0) { - /* - * OK, it begins with "usbus". - */ - long busnum; - char *p; - - errno = 0; - busnum = strtol(name + 5, &p, 10); - if (errno == 0 && p != name + 5 && *p == '\0' && - busnum >= 0 && busnum <= INT_MAX) { - /* - * OK, it's a valid number that's not - * bigger than INT_MAX. Construct - * a description from it. - */ - static const char descr_prefix[] = "USB bus number "; - size_t descr_size; - - /* - * Allow enough room for a 32-bit bus number. - * sizeof (descr_prefix) includes the - * terminating NUL. - */ - descr_size = sizeof (descr_prefix) + 10; - description = malloc(descr_size); - if (description != NULL) { - pcap_snprintf(description, descr_size, - "%s%ld", descr_prefix, busnum); - } - } - } - } -#endif - return (description); -#else /* SIOCGIFDESCR */ - return (NULL); -#endif /* SIOCGIFDESCR */ -} - -/* - * Look for a given device in the specified list of devices. - * - * If we find it, return 0 and set *curdev_ret to point to it. - * - * If we don't find it, check whether we can open it: - * - * If that fails with PCAP_ERROR_NO_SUCH_DEVICE or - * PCAP_ERROR_IFACE_NOT_UP, don't attempt to add an entry for - * it, as that probably means it exists but doesn't support - * packet capture. - * - * Otherwise, attempt to add an entry for it, with the specified - * ifnet flags and description, and, if that succeeds, return 0 - * and set *curdev_ret to point to the new entry, otherwise - * return PCAP_ERROR and set errbuf to an error message. If we - * weren't given a description, try to get one. - */ -int -add_or_find_if(pcap_if_t **curdev_ret, pcap_if_t **alldevs, const char *name, - bpf_u_int32 flags, const char *description, char *errbuf) -{ - pcap_t *p; - pcap_if_t *curdev, *prevdev, *nextdev; - u_int this_figure_of_merit, nextdev_figure_of_merit; - char open_errbuf[PCAP_ERRBUF_SIZE]; - int ret; - - /* - * Is there already an entry in the list for this interface? - */ - for (curdev = *alldevs; curdev != NULL; curdev = curdev->next) { - if (strcmp(name, curdev->name) == 0) - break; /* yes, we found it */ - } - - if (curdev == NULL) { - /* - * No, we didn't find it. - * - * Can we open this interface for live capture? - * - * We do this check so that interfaces that are - * supplied by the interface enumeration mechanism - * we're using but that don't support packet capture - * aren't included in the list. Loopback interfaces - * on Solaris are an example of this; we don't just - * omit loopback interfaces on all platforms because - * you *can* capture on loopback interfaces on some - * OSes. - * - * On OS X, we don't do this check if the device - * name begins with "wlt"; at least some versions - * of OS X offer monitor mode capturing by having - * a separate "monitor mode" device for each wireless - * adapter, rather than by implementing the ioctls - * that {Free,Net,Open,DragonFly}BSD provide. - * Opening that device puts the adapter into monitor - * mode, which, at least for some adapters, causes - * them to deassociate from the network with which - * they're associated. - * - * Instead, we try to open the corresponding "en" - * device (so that we don't end up with, for users - * without sufficient privilege to open capture - * devices, a list of adapters that only includes - * the wlt devices). - */ -#ifdef __APPLE__ - if (strncmp(name, "wlt", 3) == 0) { - char *en_name; - size_t en_name_len; - - /* - * Try to allocate a buffer for the "en" - * device's name. - */ - en_name_len = strlen(name) - 1; - en_name = malloc(en_name_len + 1); - if (en_name == NULL) { - (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "malloc: %s", pcap_strerror(errno)); - return (-1); - } - strcpy(en_name, "en"); - strcat(en_name, name + 3); - p = pcap_create(en_name, open_errbuf); - free(en_name); - } else -#endif /* __APPLE */ - p = pcap_create(name, open_errbuf); - if (p == NULL) { - /* - * The attempt to create the pcap_t failed; - * that's probably an indication that we're - * out of memory. - * - * Don't bother including this interface, - * but don't treat it as an error. - */ - *curdev_ret = NULL; - return (0); - } - /* Small snaplen, so we don't try to allocate much memory. */ - pcap_set_snaplen(p, 68); - ret = pcap_activate(p); - pcap_close(p); - switch (ret) { - - case PCAP_ERROR_NO_SUCH_DEVICE: - case PCAP_ERROR_IFACE_NOT_UP: - /* - * We expect these two errors - they're the - * reason we try to open the device. - * - * PCAP_ERROR_NO_SUCH_DEVICE typically means - * "there's no such device *known to the - * OS's capture mechanism*", so, even though - * it might be a valid network interface, you - * can't capture on it (e.g., the loopback - * device in Solaris up to Solaris 10, or - * the vmnet devices in OS X with VMware - * Fusion). We don't include those devices - * in our list of devices, as there's no - * point in doing so - they're not available - * for capture. - * - * PCAP_ERROR_IFACE_NOT_UP means that the - * OS's capture mechanism doesn't work on - * interfaces not marked as up; some capture - * mechanisms *do* support that, so we no - * longer reject those interfaces out of hand, - * but we *do* want to reject them if they - * can't be opened for capture. - */ - *curdev_ret = NULL; - return (0); - } - - /* - * Yes, we can open it, or we can't, for some other - * reason. - * - * If we can open it, we want to offer it for - * capture, as you can capture on it. If we can't, - * we want to offer it for capture, so that, if - * the user tries to capture on it, they'll get - * an error and they'll know why they can't - * capture on it (e.g., insufficient permissions) - * or they'll report it as a problem (and then - * have the error message to provide as information). - * - * Allocate a new entry. - */ - curdev = malloc(sizeof(pcap_if_t)); - if (curdev == NULL) { - (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "malloc: %s", pcap_strerror(errno)); - return (-1); - } - - /* - * Fill in the entry. - */ - curdev->next = NULL; - curdev->name = strdup(name); - if (curdev->name == NULL) { - (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "malloc: %s", pcap_strerror(errno)); - free(curdev); - return (-1); - } - if (description == NULL) { - /* - * We weren't handed a description for the - * interface, so see if we can generate one - * ourselves. - */ - curdev->description = get_if_description(name); - } else { - /* - * We were handed a description; make a copy. - */ - curdev->description = strdup(description); - if (curdev->description == NULL) { - (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "malloc: %s", pcap_strerror(errno)); - free(curdev->name); - free(curdev); - return (-1); - } - } - curdev->addresses = NULL; /* list starts out as empty */ - curdev->flags = flags; - - /* - * Add it to the list, in the appropriate location. - * First, get the "figure of merit" for this - * interface. - */ - this_figure_of_merit = get_figure_of_merit(curdev); - - /* - * Now look for the last interface with an figure of merit - * less than or equal to the new interface's figure of - * merit. - * - * We start with "prevdev" being NULL, meaning we're before - * the first element in the list. - */ - prevdev = NULL; - for (;;) { - /* - * Get the interface after this one. - */ - if (prevdev == NULL) { - /* - * The next element is the first element. - */ - nextdev = *alldevs; - } else - nextdev = prevdev->next; - - /* - * Are we at the end of the list? - */ - if (nextdev == NULL) { - /* - * Yes - we have to put the new entry - * after "prevdev". - */ - break; - } - - /* - * Is the new interface's figure of merit less - * than the next interface's figure of merit, - * meaning that the new interface is better - * than the next interface? - */ - nextdev_figure_of_merit = get_figure_of_merit(nextdev); - if (this_figure_of_merit < nextdev_figure_of_merit) { - /* - * Yes - we should put the new entry - * before "nextdev", i.e. after "prevdev". - */ - break; - } - - prevdev = nextdev; - } - - /* - * Insert before "nextdev". - */ - curdev->next = nextdev; - - /* - * Insert after "prevdev" - unless "prevdev" is null, - * in which case this is the first interface. - */ - if (prevdev == NULL) { - /* - * This is the first interface. Pass back a - * pointer to it, and put "curdev" before - * "nextdev". - */ - *alldevs = curdev; - } else - prevdev->next = curdev; - } - - *curdev_ret = curdev; - return (0); -} - -/* - * Try to get a description for a given device, and then look for that - * device in the specified list of devices. - * - * If we find it, then, if the specified address isn't null, add it to - * the list of addresses for the device and return 0. - * - * If we don't find it, check whether we can open it: - * - * If that fails with PCAP_ERROR_NO_SUCH_DEVICE or - * PCAP_ERROR_IFACE_NOT_UP, don't attempt to add an entry for - * it, as that probably means it exists but doesn't support - * packet capture. - * - * Otherwise, attempt to add an entry for it, with the specified - * ifnet flags, and, if that succeeds, add the specified address - * to its list of addresses if that address is non-null, set - * *curdev_ret to point to the new entry, and return 0, otherwise - * return PCAP_ERROR and set errbuf to an error message. - * - * (We can get called with a null address because we might get a list - * of interface name/address combinations from the underlying OS, with - * the address being absent in some cases, rather than a list of - * interfaces with each interface having a list of addresses, so this - * call may be the only call made to add to the list, and we want to - * add interfaces even if they have no addresses.) - */ -int -add_addr_to_iflist(pcap_if_t **alldevs, const char *name, bpf_u_int32 flags, - struct sockaddr *addr, size_t addr_size, - struct sockaddr *netmask, size_t netmask_size, - struct sockaddr *broadaddr, size_t broadaddr_size, - struct sockaddr *dstaddr, size_t dstaddr_size, - char *errbuf) -{ - pcap_if_t *curdev; - - if (add_or_find_if(&curdev, alldevs, name, flags, NULL, errbuf) == -1) { - /* - * Error - give up. - */ - return (-1); - } - if (curdev == NULL) { - /* - * Device wasn't added because it can't be opened. - * Not a fatal error. - */ - return (0); - } - - if (addr == NULL) { - /* - * There's no address to add; this entry just meant - * "here's a new interface". - */ - return (0); - } - - /* - * "curdev" is an entry for this interface, and we have an - * address for it; add an entry for that address to the - * interface's list of addresses. - * - * Allocate the new entry and fill it in. - */ - return (add_addr_to_dev(curdev, addr, addr_size, netmask, - netmask_size, broadaddr, broadaddr_size, dstaddr, - dstaddr_size, errbuf)); -} - -/* - * Add an entry to the list of addresses for an interface. - * "curdev" is the entry for that interface. - * If this is the first IP address added to the interface, move it - * in the list as appropriate. - */ -int -add_addr_to_dev(pcap_if_t *curdev, - struct sockaddr *addr, size_t addr_size, - struct sockaddr *netmask, size_t netmask_size, - struct sockaddr *broadaddr, size_t broadaddr_size, - struct sockaddr *dstaddr, size_t dstaddr_size, - char *errbuf) -{ - pcap_addr_t *curaddr, *prevaddr, *nextaddr; - - curaddr = malloc(sizeof(pcap_addr_t)); - if (curaddr == NULL) { - (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "malloc: %s", pcap_strerror(errno)); - return (-1); - } - - curaddr->next = NULL; - if (addr != NULL) { - curaddr->addr = dup_sockaddr(addr, addr_size); - if (curaddr->addr == NULL) { - (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "malloc: %s", pcap_strerror(errno)); - free(curaddr); - return (-1); - } - } else - curaddr->addr = NULL; - - if (netmask != NULL) { - curaddr->netmask = dup_sockaddr(netmask, netmask_size); - if (curaddr->netmask == NULL) { - (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "malloc: %s", pcap_strerror(errno)); - if (curaddr->addr != NULL) - free(curaddr->addr); - free(curaddr); - return (-1); - } - } else - curaddr->netmask = NULL; - - if (broadaddr != NULL) { - curaddr->broadaddr = dup_sockaddr(broadaddr, broadaddr_size); - if (curaddr->broadaddr == NULL) { - (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "malloc: %s", pcap_strerror(errno)); - if (curaddr->netmask != NULL) - free(curaddr->netmask); - if (curaddr->addr != NULL) - free(curaddr->addr); - free(curaddr); - return (-1); - } - } else - curaddr->broadaddr = NULL; - - if (dstaddr != NULL) { - curaddr->dstaddr = dup_sockaddr(dstaddr, dstaddr_size); - if (curaddr->dstaddr == NULL) { - (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "malloc: %s", pcap_strerror(errno)); - if (curaddr->broadaddr != NULL) - free(curaddr->broadaddr); - if (curaddr->netmask != NULL) - free(curaddr->netmask); - if (curaddr->addr != NULL) - free(curaddr->addr); - free(curaddr); - return (-1); - } - } else - curaddr->dstaddr = NULL; - - /* - * Find the end of the list of addresses. - */ - for (prevaddr = curdev->addresses; prevaddr != NULL; prevaddr = nextaddr) { - nextaddr = prevaddr->next; - if (nextaddr == NULL) { - /* - * This is the end of the list. - */ - break; - } - } - - if (prevaddr == NULL) { - /* - * The list was empty; this is the first member. - */ - curdev->addresses = curaddr; - } else { - /* - * "prevaddr" is the last member of the list; append - * this member to it. - */ - prevaddr->next = curaddr; - } - - return (0); -} - -/* - * Look for a given device in the specified list of devices. - * - * If we find it, return 0. - * - * If we don't find it, check whether we can open it: - * - * If that fails with PCAP_ERROR_NO_SUCH_DEVICE or - * PCAP_ERROR_IFACE_NOT_UP, don't attempt to add an entry for - * it, as that probably means it exists but doesn't support - * packet capture. - * - * Otherwise, attempt to add an entry for it, with the specified - * ifnet flags and description, and, if that succeeds, return 0 - * and set *curdev_ret to point to the new entry, otherwise - * return PCAP_ERROR and set errbuf to an error message. - */ -int -#ifndef __rtems__ -pcap_add_if(pcap_if_t **devlist, const char *name, u_int flags, -#else /* __rtems__ */ -pcap_add_if(pcap_if_t **devlist, const char *name, bpf_u_int32 flags, -#endif /* __rtems__ */ - const char *description, char *errbuf) -{ - pcap_if_t *curdev; - - return (add_or_find_if(&curdev, devlist, name, flags, description, - errbuf)); -} - - -/* - * Free a list of interfaces. - */ -void -pcap_freealldevs(pcap_if_t *alldevs) -{ - pcap_if_t *curdev, *nextdev; - pcap_addr_t *curaddr, *nextaddr; - - for (curdev = alldevs; curdev != NULL; curdev = nextdev) { - nextdev = curdev->next; - - /* - * Free all addresses. - */ - for (curaddr = curdev->addresses; curaddr != NULL; curaddr = nextaddr) { - nextaddr = curaddr->next; - if (curaddr->addr) - free(curaddr->addr); - if (curaddr->netmask) - free(curaddr->netmask); - if (curaddr->broadaddr) - free(curaddr->broadaddr); - if (curaddr->dstaddr) - free(curaddr->dstaddr); - free(curaddr); - } - - /* - * Free the name string. - */ - free(curdev->name); - - /* - * Free the description string, if any. - */ - if (curdev->description != NULL) - free(curdev->description); - - /* - * Free the interface. - */ - free(curdev); - } -} diff --git a/freebsd/contrib/libpcap/fmtutils.c b/freebsd/contrib/libpcap/fmtutils.c new file mode 100644 index 00000000..ae550e24 --- /dev/null +++ b/freebsd/contrib/libpcap/fmtutils.c @@ -0,0 +1,133 @@ +#include <machine/rtems-bsd-user-space.h> + +/* + * Copyright (c) 1993, 1994, 1995, 1996, 1997, 1998 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the Computer Systems + * Engineering Group at Lawrence Berkeley Laboratory. + * 4. Neither the name of the University nor of the Laboratory may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Utilities for message formatting used both by libpcap and rpcapd. + */ + +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif + +#include "ftmacros.h" + +#include <stddef.h> +#include <stdarg.h> +#include <stdio.h> +#include <string.h> +#include <errno.h> + +#include <pcap/pcap.h> + +#include "portability.h" + +#include "fmtutils.h" + +/* + * Generate an error message based on a format, arguments, and an + * errno, with a message for the errno after the formatted output. + */ +void +pcap_fmt_errmsg_for_errno(char *errbuf, size_t errbuflen, int errnum, + const char *fmt, ...) +{ + va_list ap; + size_t msglen; + char *p; + size_t errbuflen_remaining; +#if defined(HAVE_STRERROR_S) + errno_t err; +#elif defined(HAVE_STRERROR_R) + int err; +#endif + + va_start(ap, fmt); + pcap_vsnprintf(errbuf, errbuflen, fmt, ap); + va_end(ap); + msglen = strlen(errbuf); + + /* + * Do we have enough space to append ": "? + * Including the terminating '\0', that's 3 bytes. + */ + if (msglen + 3 > errbuflen) { + /* No - just give them what we've produced. */ + return; + } + p = errbuf + msglen; + errbuflen_remaining = errbuflen - msglen; + *p++ = ':'; + *p++ = ' '; + *p = '\0'; + msglen += 2; + errbuflen_remaining -= 2; + + /* + * Now append the string for the error code. + */ +#if defined(HAVE_STRERROR_S) + err = strerror_s(p, errbuflen_remaining, errnum); + if (err != 0) { + /* + * It doesn't appear to be documented anywhere obvious + * what the error returns from strerror_s(). + */ + pcap_snprintf(p, errbuflen_remaining, "Error %d", errnum); + } +#elif defined(HAVE_STRERROR_R) + err = strerror_r(errnum, p, errbuflen_remaining); + if (err == EINVAL) { + /* + * UNIX 03 says this isn't guaranteed to produce a + * fallback error message. + */ + pcap_snprintf(p, errbuflen_remaining, "Unknown error: %d", + errnum); + } else if (err == ERANGE) { + /* + * UNIX 03 says this isn't guaranteed to produce a + * fallback error message. + */ + pcap_snprintf(p, errbuflen_remaining, + "Message for error %d is too long", errnum); + } +#else + /* + * We have neither strerror_s() nor strerror_r(), so we're + * stuck with using pcap_strerror(). + */ + pcap_snprintf(p, errbuflen_remaining, "%s", pcap_strerror(errnum)); +#endif +} diff --git a/freebsd/contrib/libpcap/fmtutils.h b/freebsd/contrib/libpcap/fmtutils.h new file mode 100644 index 00000000..62c78fdb --- /dev/null +++ b/freebsd/contrib/libpcap/fmtutils.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 1994, 1995, 1996 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the Computer Systems + * Engineering Group at Lawrence Berkeley Laboratory. + * 4. Neither the name of the University nor of the Laboratory may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef fmtutils_h +#define fmtutils_h + +#include "pcap/funcattrs.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void pcap_fmt_errmsg_for_errno(char *, size_t, int, + PCAP_FORMAT_STRING(const char *), ...) PCAP_PRINTFLIKE(4, 5); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/freebsd/contrib/libpcap/ftmacros.h b/freebsd/contrib/libpcap/ftmacros.h new file mode 100644 index 00000000..de8da98e --- /dev/null +++ b/freebsd/contrib/libpcap/ftmacros.h @@ -0,0 +1,115 @@ +/* + * Copyright (c) 1994, 1995, 1996 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the Computer Systems + * Engineering Group at Lawrence Berkeley Laboratory. + * 4. Neither the name of the University nor of the Laboratory may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef ftmacros_h +#define ftmacros_h + +/* + * Define some feature test macros to make sure that everything we want + * to be declared gets declared. + * + * On some UN*Xes we need to force strtok_r() to be declared. + * We do *NOT* want to define _POSIX_C_SOURCE, as that tends + * to make non-POSIX APIs that we use unavailable. + * XXX - is there no portable way to say "please pollute the + * namespace to the maximum extent possible"? + */ +#if defined(sun) || defined(__sun) + #define __EXTENSIONS__ + + /* + * We also need to define _XPG4_2 in order to get + * the Single UNIX Specification version of + * recvmsg(). + */ + #define _XPG4_2 +#elif defined(_hpux) || defined(hpux) || defined(__hpux) + #define _REENTRANT + + /* + * We need this to get the versions of socket functions that + * use socklen_t. Define it only if it's not already defined, + * so we don't get redefiniton warnings. + */ + #ifndef _XOPEN_SOURCE_EXTENDED + #define _XOPEN_SOURCE_EXTENDED + #endif + + /* + * XXX - the list of PA-RISC options for GCC makes it sound as if + * building code that uses a particular vintage of UNIX API/ABI + * is complicated: + * + * https://gcc.gnu.org/onlinedocs/gcc/HPPA-Options.html + * + * See the description of the -munix flag. + * + * We probably want libpcap to work with programs built for any + * UN*X standard. I'm not sure whether that's possible and, if + * it is, what sort of stuff it'd have to do. + * + * It might also be a requirement that we build with a special + * flag to allow the library to be used with threaded code, at + * least with HP's C compiler; hopefully doing so won't make it + * *not* work with *un*-threaded code. + */ +#elif defined(__linux__) || defined(linux) || defined(__linux) + /* + * We can't turn _GNU_SOURCE on because some versions of GNU Libc + * will give the GNU version of strerror_r(), which returns a + * string pointer and doesn't necessarily fill in the buffer, + * rather than the standard version of strerror_r(), which + * returns 0 or an errno and always fills in the buffer. We + * require both of the latter behaviors. + * + * So we try turning everything else on that we can. This includes + * defining _XOPEN_SOURCE as 600, because we want to force crypt() + * to be declared on systems that use GNU libc, such as most Linux + * distributions. + */ + #define _POSIX_C_SOURCE 200809L + #define _XOPEN_SOURCE 600 + + /* + * We turn on both _DEFAULT_SOURCE and _BSD_SOURCE to try to get + * the BSD u_XXX types, such as u_int and u_short, defined. We + * define _DEFAULT_SOURCE first, so that newer versions of GNU libc + * don't whine about _BSD_SOURCE being deprecated; we still have + * to define _BSD_SOURCE to handle older versions of GNU libc that + * don't support _DEFAULT_SOURCE. + */ + #define _DEFAULT_SOURCE + #define _BSD_SOURCE +#endif + +#endif diff --git a/freebsd/contrib/libpcap/gencode.c b/freebsd/contrib/libpcap/gencode.c index 8c303415..df75cacd 100644 --- a/freebsd/contrib/libpcap/gencode.c +++ b/freebsd/contrib/libpcap/gencode.c @@ -20,38 +20,24 @@ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. - * - * $FreeBSD$ */ #ifdef HAVE_CONFIG_H -#include "config.h" +#include <config.h> #endif +#include <pcap-types.h> #ifdef _WIN32 -#include <pcap-stdinc.h> -#else /* _WIN32 */ -#if HAVE_INTTYPES_H -#include <inttypes.h> -#elif HAVE_STDINT_H -#include <stdint.h> -#endif -#ifdef HAVE_SYS_BITYPES_H -#include <sys/bitypes.h> -#endif -#include <sys/types.h> -#include <sys/socket.h> -#endif /* _WIN32 */ - -#ifndef _WIN32 - -#ifdef __NetBSD__ -#include <sys/param.h> -#endif + #include <ws2tcpip.h> +#else + #include <sys/socket.h> -#include <netinet/in.h> -#include <arpa/inet.h> + #ifdef __NetBSD__ + #include <sys/param.h> + #endif + #include <netinet/in.h> + #include <arpa/inet.h> #endif /* _WIN32 */ #include <stdlib.h> @@ -98,17 +84,17 @@ #define offsetof(s, e) ((size_t)&((s *)0)->e) #endif -#ifdef INET6 #ifdef _WIN32 -#if defined(__MINGW32__) && defined(DEFINE_ADDITIONAL_IPV6_STUFF) + #ifdef INET6 + #if defined(__MINGW32__) && defined(DEFINE_ADDITIONAL_IPV6_STUFF) /* IPv6 address */ struct in6_addr { union { - u_int8_t u6_addr8[16]; - u_int16_t u6_addr16[8]; - u_int32_t u6_addr32[4]; + uint8_t u6_addr8[16]; + uint16_t u6_addr16[8]; + uint32_t u6_addr32[4]; } in6_u; #define s6_addr in6_u.u6_addr8 #define s6_addr16 in6_u.u6_addr16 @@ -125,12 +111,12 @@ typedef unsigned short sa_family_t; struct sockaddr_in6 { __SOCKADDR_COMMON (sin6_); - u_int16_t sin6_port; /* Transport layer port # */ - u_int32_t sin6_flowinfo; /* IPv6 flow information */ + uint16_t sin6_port; /* Transport layer port # */ + uint32_t sin6_flowinfo; /* IPv6 flow information */ struct in6_addr sin6_addr; /* IPv6 address */ }; -#ifndef EAI_ADDRFAMILY + #ifndef EAI_ADDRFAMILY struct addrinfo { int ai_flags; /* AI_PASSIVE, AI_CANONNAME */ int ai_family; /* PF_xxx */ @@ -141,12 +127,12 @@ struct addrinfo { struct sockaddr *ai_addr; /* binary address */ struct addrinfo *ai_next; /* next structure in linked list */ }; -#endif /* EAI_ADDRFAMILY */ -#endif /* defined(__MINGW32__) && defined(DEFINE_ADDITIONAL_IPV6_STUFF) */ + #endif /* EAI_ADDRFAMILY */ + #endif /* defined(__MINGW32__) && defined(DEFINE_ADDITIONAL_IPV6_STUFF) */ + #endif /* INET6 */ #else /* _WIN32 */ -#include <netdb.h> /* for "struct addrinfo" */ + #include <netdb.h> /* for "struct addrinfo" */ #endif /* _WIN32 */ -#endif /* INET6 */ #include <pcap/namedb.h> #include "nametoaddr.h" @@ -281,7 +267,6 @@ struct _compiler_state { /* XXX */ u_int pcap_fddipad; -#ifdef INET6 /* * As errors are handled by a longjmp, anything allocated must * be freed in the longjmp handler, so it must be reachable @@ -292,7 +277,6 @@ struct _compiler_state { * any addrinfo structure that would need to be freed. */ struct addrinfo *ai; -#endif /* * Various code constructs need to know the layout of the packet. @@ -362,6 +346,11 @@ struct _compiler_state { int is_geneve; /* + * TRUE if we need variable length part of VLAN offset + */ + int is_vlan_vloffset; + + /* * These are offsets for the ATM pseudo-header. */ u_int off_vpi; @@ -430,7 +419,7 @@ struct _compiler_state { int cur_chunk; }; -void +void PCAP_NORETURN bpf_syntax_error(compiler_state_t *cstate, const char *msg) { bpf_error(cstate, "syntax error in filter expression: %s", msg); @@ -438,7 +427,7 @@ bpf_syntax_error(compiler_state_t *cstate, const char *msg) } /* VARARGS */ -void +void PCAP_NORETURN bpf_error(compiler_state_t *cstate, const char *fmt, ...) { va_list ap; @@ -528,7 +517,7 @@ static struct block *gen_host6(compiler_state_t *, struct in6_addr *, #endif #ifndef INET6 static struct block *gen_gateway(compiler_state_t *, const u_char *, - bpf_u_int32 **, int, int); + struct addrinfo *, int, int); #endif static struct block *gen_ipfrag(compiler_state_t *); static struct block *gen_portatom(compiler_state_t *, int, bpf_int32); @@ -587,7 +576,8 @@ newchunk(compiler_state_t *cstate, size_t n) cp = &cstate->chunks[cstate->cur_chunk]; if (n > cp->n_left) { - ++cp, k = ++cstate->cur_chunk; + ++cp; + k = ++cstate->cur_chunk; if (k >= NCHUNKS) bpf_error(cstate, "out of memory"); size = CHUNK0SIZE << k; @@ -658,7 +648,7 @@ gen_retblk(compiler_state_t *cstate, int v) return b; } -static inline void +static inline PCAP_NORETURN_DEF void syntax(compiler_state_t *cstate) { bpf_error(cstate, "syntax error in filter expression"); @@ -668,6 +658,9 @@ int pcap_compile(pcap_t *p, struct bpf_program *program, const char *buf, int optimize, bpf_u_int32 mask) { +#ifdef _WIN32 + static int done = 0; +#endif compiler_state_t cstate; const char * volatile xbuf = buf; yyscan_t scanner = NULL; @@ -675,14 +668,6 @@ pcap_compile(pcap_t *p, struct bpf_program *program, u_int len; int rc; -#ifdef _WIN32 - static int done = 0; - - if (!done) - pcap_wsockinit(); - done = 1; -#endif - /* * If this pcap_t hasn't been activated, it doesn't have a * link-layer type, so we can't use it. @@ -690,9 +675,36 @@ pcap_compile(pcap_t *p, struct bpf_program *program, if (!p->activated) { pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "not-yet-activated pcap_t passed to pcap_compile"); - rc = -1; - goto quit; + return (-1); } + +#ifdef _WIN32 + if (!done) + pcap_wsockinit(); + done = 1; +#endif + +#ifdef ENABLE_REMOTE + /* + * If the device on which we're capturing need to be notified + * that a new filter is being compiled, do so. + * + * This allows them to save a copy of it, in case, for example, + * they're implementing a form of remote packet capture, and + * want the remote machine to filter out the packets in which + * it's sending the packets it's captured. + * + * XXX - the fact that we happen to be compiling a filter + * doesn't necessarily mean we'll be installing it as the + * filter for this pcap_t; we might be running it from userland + * on captured packets to do packet classification. We really + * need a better way of handling this, but this is all that + * the WinPcap code did. + */ + if (p->save_current_filter_op != NULL) + (p->save_current_filter_op)(p, buf); +#endif + initchunks(&cstate); cstate.no_optimize = 0; #ifdef INET6 @@ -723,7 +735,8 @@ pcap_compile(pcap_t *p, struct bpf_program *program, } if (pcap_lex_init(&scanner) != 0) - bpf_error(&cstate, "can't initialize scanner: %s", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "can't initialize scanner"); in_buffer = pcap__scan_string(xbuf ? xbuf : "", scanner); /* @@ -807,8 +820,7 @@ pcap_freecode(struct bpf_program *program) * in each block is already resolved. */ static void -backpatch(list, target) - struct block *list, *target; +backpatch(struct block *list, struct block *target) { struct block *next; @@ -829,8 +841,7 @@ backpatch(list, target) * which of jt and jf is the link. */ static void -merge(b0, b1) - struct block *b0, *b1; +merge(struct block *b0, struct block *b1) { register struct block **p = &b0; @@ -892,8 +903,7 @@ finish_parse(compiler_state_t *cstate, struct block *p) } void -gen_and(b0, b1) - struct block *b0, *b1; +gen_and(struct block *b0, struct block *b1) { backpatch(b0, b1->head); b0->sense = !b0->sense; @@ -904,8 +914,7 @@ gen_and(b0, b1) } void -gen_or(b0, b1) - struct block *b0, *b1; +gen_or(struct block *b0, struct block *b1) { b0->sense = !b0->sense; backpatch(b0, b1->head); @@ -915,8 +924,7 @@ gen_or(b0, b1) } void -gen_not(b) - struct block *b; +gen_not(struct block *b) { b->sense = !b->sense; } @@ -1068,10 +1076,10 @@ init_linktype(compiler_state_t *cstate, pcap_t *p) * Assume it's not raw ATM with a pseudo-header, for now. */ cstate->is_atm = 0; - cstate->off_vpi = -1; - cstate->off_vci = -1; - cstate->off_proto = -1; - cstate->off_payload = -1; + cstate->off_vpi = OFFSET_NOT_SET; + cstate->off_vci = OFFSET_NOT_SET; + cstate->off_proto = OFFSET_NOT_SET; + cstate->off_payload = OFFSET_NOT_SET; /* * And not Geneve. @@ -1079,14 +1087,19 @@ init_linktype(compiler_state_t *cstate, pcap_t *p) cstate->is_geneve = 0; /* + * No variable length VLAN offset by default + */ + cstate->is_vlan_vloffset = 0; + + /* * And assume we're not doing SS7. */ - cstate->off_li = -1; - cstate->off_li_hsl = -1; - cstate->off_sio = -1; - cstate->off_opc = -1; - cstate->off_dpc = -1; - cstate->off_sls = -1; + cstate->off_li = OFFSET_NOT_SET; + cstate->off_li_hsl = OFFSET_NOT_SET; + cstate->off_sio = OFFSET_NOT_SET; + cstate->off_opc = OFFSET_NOT_SET; + cstate->off_dpc = OFFSET_NOT_SET; + cstate->off_sls = OFFSET_NOT_SET; cstate->label_stack_depth = 0; cstate->vlan_stack_depth = 0; @@ -1096,7 +1109,7 @@ init_linktype(compiler_state_t *cstate, pcap_t *p) case DLT_ARCNET: cstate->off_linktype.constant_part = 2; cstate->off_linkpl.constant_part = 6; - cstate->off_nl = 0; /* XXX in reality, variable! */ + cstate->off_nl = 0; /* XXX in reality, variable! */ cstate->off_nl_nosnap = 0; /* no 802.2 LLC */ break; @@ -1410,7 +1423,7 @@ init_linktype(compiler_state_t *cstate, pcap_t *p) cstate->off_linktype.constant_part = 4; cstate->off_linkpl.constant_part = 4; cstate->off_nl = 0; - cstate->off_nl_nosnap = -1; /* no 802.2 LLC */ + cstate->off_nl_nosnap = OFFSET_NOT_SET; /* no 802.2 LLC */ break; case DLT_JUNIPER_ATM1: @@ -1441,63 +1454,63 @@ init_linktype(compiler_state_t *cstate, pcap_t *p) cstate->off_linktype.constant_part = 4; cstate->off_linkpl.constant_part = 6; cstate->off_nl = 0; - cstate->off_nl_nosnap = -1; /* no 802.2 LLC */ + cstate->off_nl_nosnap = OFFSET_NOT_SET; /* no 802.2 LLC */ break; case DLT_JUNIPER_GGSN: cstate->off_linktype.constant_part = 6; cstate->off_linkpl.constant_part = 12; cstate->off_nl = 0; - cstate->off_nl_nosnap = -1; /* no 802.2 LLC */ + cstate->off_nl_nosnap = OFFSET_NOT_SET; /* no 802.2 LLC */ break; case DLT_JUNIPER_ES: cstate->off_linktype.constant_part = 6; cstate->off_linkpl.constant_part = OFFSET_NOT_SET; /* not really a network layer but raw IP addresses */ - cstate->off_nl = -1; /* not really a network layer but raw IP addresses */ - cstate->off_nl_nosnap = -1; /* no 802.2 LLC */ + cstate->off_nl = OFFSET_NOT_SET; /* not really a network layer but raw IP addresses */ + cstate->off_nl_nosnap = OFFSET_NOT_SET; /* no 802.2 LLC */ break; case DLT_JUNIPER_MONITOR: cstate->off_linktype.constant_part = 12; cstate->off_linkpl.constant_part = 12; - cstate->off_nl = 0; /* raw IP/IP6 header */ - cstate->off_nl_nosnap = -1; /* no 802.2 LLC */ + cstate->off_nl = 0; /* raw IP/IP6 header */ + cstate->off_nl_nosnap = OFFSET_NOT_SET; /* no 802.2 LLC */ break; case DLT_BACNET_MS_TP: cstate->off_linktype.constant_part = OFFSET_NOT_SET; cstate->off_linkpl.constant_part = OFFSET_NOT_SET; - cstate->off_nl = -1; - cstate->off_nl_nosnap = -1; + cstate->off_nl = OFFSET_NOT_SET; + cstate->off_nl_nosnap = OFFSET_NOT_SET; break; case DLT_JUNIPER_SERVICES: cstate->off_linktype.constant_part = 12; cstate->off_linkpl.constant_part = OFFSET_NOT_SET; /* L3 proto location dep. on cookie type */ - cstate->off_nl = -1; /* L3 proto location dep. on cookie type */ - cstate->off_nl_nosnap = -1; /* no 802.2 LLC */ + cstate->off_nl = OFFSET_NOT_SET; /* L3 proto location dep. on cookie type */ + cstate->off_nl_nosnap = OFFSET_NOT_SET; /* no 802.2 LLC */ break; case DLT_JUNIPER_VP: cstate->off_linktype.constant_part = 18; cstate->off_linkpl.constant_part = OFFSET_NOT_SET; - cstate->off_nl = -1; - cstate->off_nl_nosnap = -1; + cstate->off_nl = OFFSET_NOT_SET; + cstate->off_nl_nosnap = OFFSET_NOT_SET; break; case DLT_JUNIPER_ST: cstate->off_linktype.constant_part = 18; cstate->off_linkpl.constant_part = OFFSET_NOT_SET; - cstate->off_nl = -1; - cstate->off_nl_nosnap = -1; + cstate->off_nl = OFFSET_NOT_SET; + cstate->off_nl_nosnap = OFFSET_NOT_SET; break; case DLT_JUNIPER_ISM: cstate->off_linktype.constant_part = 8; cstate->off_linkpl.constant_part = OFFSET_NOT_SET; - cstate->off_nl = -1; - cstate->off_nl_nosnap = -1; + cstate->off_nl = OFFSET_NOT_SET; + cstate->off_nl_nosnap = OFFSET_NOT_SET; break; case DLT_JUNIPER_VS: @@ -1506,8 +1519,8 @@ init_linktype(compiler_state_t *cstate, pcap_t *p) case DLT_JUNIPER_ATM_CEMIC: cstate->off_linktype.constant_part = 8; cstate->off_linkpl.constant_part = OFFSET_NOT_SET; - cstate->off_nl = -1; - cstate->off_nl_nosnap = -1; + cstate->off_nl = OFFSET_NOT_SET; + cstate->off_nl_nosnap = OFFSET_NOT_SET; break; case DLT_MTP2: @@ -1519,8 +1532,8 @@ init_linktype(compiler_state_t *cstate, pcap_t *p) cstate->off_sls = 7; cstate->off_linktype.constant_part = OFFSET_NOT_SET; cstate->off_linkpl.constant_part = OFFSET_NOT_SET; - cstate->off_nl = -1; - cstate->off_nl_nosnap = -1; + cstate->off_nl = OFFSET_NOT_SET; + cstate->off_nl_nosnap = OFFSET_NOT_SET; break; case DLT_MTP2_WITH_PHDR: @@ -1532,8 +1545,8 @@ init_linktype(compiler_state_t *cstate, pcap_t *p) cstate->off_sls = 11; cstate->off_linktype.constant_part = OFFSET_NOT_SET; cstate->off_linkpl.constant_part = OFFSET_NOT_SET; - cstate->off_nl = -1; - cstate->off_nl_nosnap = -1; + cstate->off_nl = OFFSET_NOT_SET; + cstate->off_nl_nosnap = OFFSET_NOT_SET; break; case DLT_ERF: @@ -1545,8 +1558,8 @@ init_linktype(compiler_state_t *cstate, pcap_t *p) cstate->off_sls = 27; cstate->off_linktype.constant_part = OFFSET_NOT_SET; cstate->off_linkpl.constant_part = OFFSET_NOT_SET; - cstate->off_nl = -1; - cstate->off_nl_nosnap = -1; + cstate->off_nl = OFFSET_NOT_SET; + cstate->off_nl_nosnap = OFFSET_NOT_SET; break; case DLT_PFSYNC: @@ -1562,15 +1575,15 @@ init_linktype(compiler_state_t *cstate, pcap_t *p) */ cstate->off_linktype.constant_part = OFFSET_NOT_SET; /* variable, min 15, max 71 steps of 7 */ cstate->off_linkpl.constant_part = OFFSET_NOT_SET; - cstate->off_nl = -1; /* variable, min 16, max 71 steps of 7 */ - cstate->off_nl_nosnap = -1; /* no 802.2 LLC */ + cstate->off_nl = OFFSET_NOT_SET; /* variable, min 16, max 71 steps of 7 */ + cstate->off_nl_nosnap = OFFSET_NOT_SET; /* no 802.2 LLC */ break; case DLT_IPNET: cstate->off_linktype.constant_part = 1; cstate->off_linkpl.constant_part = 24; /* ipnet header length */ cstate->off_nl = 0; - cstate->off_nl_nosnap = -1; + cstate->off_nl_nosnap = OFFSET_NOT_SET; break; case DLT_NETANALYZER: @@ -1598,8 +1611,8 @@ init_linktype(compiler_state_t *cstate, pcap_t *p) cstate->linktype <= DLT_MATCHING_MAX) { cstate->off_linktype.constant_part = OFFSET_NOT_SET; cstate->off_linkpl.constant_part = OFFSET_NOT_SET; - cstate->off_nl = -1; - cstate->off_nl_nosnap = -1; + cstate->off_nl = OFFSET_NOT_SET; + cstate->off_nl_nosnap = OFFSET_NOT_SET; } else { bpf_error(cstate, "unknown data link type %d", cstate->linktype); } @@ -1724,7 +1737,7 @@ gen_load_a(compiler_state_t *cstate, enum e_offrel offrel, u_int offset, default: abort(); - return NULL; + /* NOTREACHED */ } return s; } @@ -1788,6 +1801,7 @@ gen_loadx_iphdrlen(compiler_state_t *cstate) return s; } + static struct block * gen_uncond(compiler_state_t *cstate, int rsense) { @@ -2805,6 +2819,28 @@ insert_compute_vloffsets(compiler_state_t *cstate, struct block *b) } /* + * If there there is no initialization yet and we need variable + * length offsets for VLAN, initialize them to zero + */ + if (s == NULL && cstate->is_vlan_vloffset) { + struct slist *s2; + + if (cstate->off_linkpl.reg == -1) + cstate->off_linkpl.reg = alloc_reg(cstate); + if (cstate->off_linktype.reg == -1) + cstate->off_linktype.reg = alloc_reg(cstate); + + s = new_stmt(cstate, BPF_LD|BPF_W|BPF_IMM); + s->s.k = 0; + s2 = new_stmt(cstate, BPF_ST); + s2->s.k = cstate->off_linkpl.reg; + sappend(s, s2); + s2 = new_stmt(cstate, BPF_ST); + s2->s.k = cstate->off_linktype.reg; + sappend(s, s2); + } + + /* * If we have any offset-loading code, append all the * existing statements in the block to those statements, * and make the resulting list the list of statements @@ -2890,8 +2926,7 @@ gen_abs_offset_varpart(compiler_state_t *cstate, bpf_abs_offset *off) * Map an Ethernet type to the equivalent PPP type. */ static int -ethertype_to_ppptype(proto) - int proto; +ethertype_to_ppptype(int proto) { switch (proto) { @@ -2978,7 +3013,7 @@ gen_prevlinkhdr_check(compiler_state_t *cstate) */ #define BSD_AFNUM_INET6_BSD 24 /* NetBSD, OpenBSD, BSD/OS, Npcap */ #define BSD_AFNUM_INET6_FREEBSD 28 /* FreeBSD */ -#define BSD_AFNUM_INET6_DARWIN 30 /* OS X, iOS, other Darwin-based OSes */ +#define BSD_AFNUM_INET6_DARWIN 30 /* macOS, iOS, other Darwin-based OSes */ /* * Generate code to match a particular packet type by matching the @@ -3534,14 +3569,14 @@ gen_snap(compiler_state_t *cstate, bpf_u_int32 orgcode, bpf_u_int32 ptype) { u_char snapblock[8]; - snapblock[0] = LLCSAP_SNAP; /* DSAP = SNAP */ - snapblock[1] = LLCSAP_SNAP; /* SSAP = SNAP */ - snapblock[2] = 0x03; /* control = UI */ - snapblock[3] = (orgcode >> 16); /* upper 8 bits of organization code */ - snapblock[4] = (orgcode >> 8); /* middle 8 bits of organization code */ - snapblock[5] = (orgcode >> 0); /* lower 8 bits of organization code */ - snapblock[6] = (ptype >> 8); /* upper 8 bits of protocol type */ - snapblock[7] = (ptype >> 0); /* lower 8 bits of protocol type */ + snapblock[0] = LLCSAP_SNAP; /* DSAP = SNAP */ + snapblock[1] = LLCSAP_SNAP; /* SSAP = SNAP */ + snapblock[2] = 0x03; /* control = UI */ + snapblock[3] = (u_char)(orgcode >> 16); /* upper 8 bits of organization code */ + snapblock[4] = (u_char)(orgcode >> 8); /* middle 8 bits of organization code */ + snapblock[5] = (u_char)(orgcode >> 0); /* lower 8 bits of organization code */ + snapblock[6] = (u_char)(ptype >> 8); /* upper 8 bits of protocol type */ + snapblock[7] = (u_char)(ptype >> 0); /* lower 8 bits of protocol type */ return gen_bcmp(cstate, OR_LLC, 0, 8, snapblock); } @@ -3834,6 +3869,30 @@ gen_hostop(compiler_state_t *cstate, bpf_u_int32 addr, bpf_u_int32 mask, gen_or(b0, b1); return b1; + case Q_ADDR1: + bpf_error(cstate, "'addr1' and 'address1' are not valid qualifiers for addresses other than 802.11 MAC addresses"); + break; + + case Q_ADDR2: + bpf_error(cstate, "'addr2' and 'address2' are not valid qualifiers for addresses other than 802.11 MAC addresses"); + break; + + case Q_ADDR3: + bpf_error(cstate, "'addr3' and 'address3' are not valid qualifiers for addresses other than 802.11 MAC addresses"); + break; + + case Q_ADDR4: + bpf_error(cstate, "'addr4' and 'address4' are not valid qualifiers for addresses other than 802.11 MAC addresses"); + break; + + case Q_RA: + bpf_error(cstate, "'ra' is not a valid qualifier for addresses other than 802.11 MAC addresses"); + break; + + case Q_TA: + bpf_error(cstate, "'ta' is not a valid qualifier for addresses other than 802.11 MAC addresses"); + break; + default: abort(); } @@ -3850,7 +3909,7 @@ gen_hostop6(compiler_state_t *cstate, struct in6_addr *addr, { struct block *b0, *b1; u_int offset; - u_int32_t *a, *m; + uint32_t *a, *m; switch (dir) { @@ -3875,12 +3934,36 @@ gen_hostop6(compiler_state_t *cstate, struct in6_addr *addr, gen_or(b0, b1); return b1; + case Q_ADDR1: + bpf_error(cstate, "'addr1' and 'address1' are not valid qualifiers for addresses other than 802.11 MAC addresses"); + break; + + case Q_ADDR2: + bpf_error(cstate, "'addr2' and 'address2' are not valid qualifiers for addresses other than 802.11 MAC addresses"); + break; + + case Q_ADDR3: + bpf_error(cstate, "'addr3' and 'address3' are not valid qualifiers for addresses other than 802.11 MAC addresses"); + break; + + case Q_ADDR4: + bpf_error(cstate, "'addr4' and 'address4' are not valid qualifiers for addresses other than 802.11 MAC addresses"); + break; + + case Q_RA: + bpf_error(cstate, "'ra' is not a valid qualifier for addresses other than 802.11 MAC addresses"); + break; + + case Q_TA: + bpf_error(cstate, "'ta' is not a valid qualifier for addresses other than 802.11 MAC addresses"); + break; + default: abort(); } /* this order is important */ - a = (u_int32_t *)addr; - m = (u_int32_t *)mask; + a = (uint32_t *)addr; + m = (uint32_t *)mask; b1 = gen_mcmp(cstate, OR_LINKPL, offset + 12, BPF_W, ntohl(a[3]), ntohl(m[3])); b0 = gen_mcmp(cstate, OR_LINKPL, offset + 8, BPF_W, ntohl(a[2]), ntohl(m[2])); gen_and(b0, b1); @@ -3920,19 +4003,19 @@ gen_ehostop(compiler_state_t *cstate, const u_char *eaddr, int dir) return b1; case Q_ADDR1: - bpf_error(cstate, "'addr1' is only supported on 802.11 with 802.11 headers"); + bpf_error(cstate, "'addr1' and 'address1' are only supported on 802.11 with 802.11 headers"); break; case Q_ADDR2: - bpf_error(cstate, "'addr2' is only supported on 802.11 with 802.11 headers"); + bpf_error(cstate, "'addr2' and 'address2' are only supported on 802.11 with 802.11 headers"); break; case Q_ADDR3: - bpf_error(cstate, "'addr3' is only supported on 802.11 with 802.11 headers"); + bpf_error(cstate, "'addr3' and 'address3' are only supported on 802.11 with 802.11 headers"); break; case Q_ADDR4: - bpf_error(cstate, "'addr4' is only supported on 802.11 with 802.11 headers"); + bpf_error(cstate, "'addr4' and 'address4' are only supported on 802.11 with 802.11 headers"); break; case Q_RA: @@ -3976,19 +4059,19 @@ gen_fhostop(compiler_state_t *cstate, const u_char *eaddr, int dir) return b1; case Q_ADDR1: - bpf_error(cstate, "'addr1' is only supported on 802.11"); + bpf_error(cstate, "'addr1' and 'address1' are only supported on 802.11"); break; case Q_ADDR2: - bpf_error(cstate, "'addr2' is only supported on 802.11"); + bpf_error(cstate, "'addr2' and 'address2' are only supported on 802.11"); break; case Q_ADDR3: - bpf_error(cstate, "'addr3' is only supported on 802.11"); + bpf_error(cstate, "'addr3' and 'address3' are only supported on 802.11"); break; case Q_ADDR4: - bpf_error(cstate, "'addr4' is only supported on 802.11"); + bpf_error(cstate, "'addr4' and 'address4' are only supported on 802.11"); break; case Q_RA: @@ -4032,19 +4115,19 @@ gen_thostop(compiler_state_t *cstate, const u_char *eaddr, int dir) return b1; case Q_ADDR1: - bpf_error(cstate, "'addr1' is only supported on 802.11"); + bpf_error(cstate, "'addr1' and 'address1' are only supported on 802.11"); break; case Q_ADDR2: - bpf_error(cstate, "'addr2' is only supported on 802.11"); + bpf_error(cstate, "'addr2' and 'address2' are only supported on 802.11"); break; case Q_ADDR3: - bpf_error(cstate, "'addr3' is only supported on 802.11"); + bpf_error(cstate, "'addr3' and 'address3' are only supported on 802.11"); break; case Q_ADDR4: - bpf_error(cstate, "'addr4' is only supported on 802.11"); + bpf_error(cstate, "'addr4' and 'address4' are only supported on 802.11"); break; case Q_RA: @@ -4513,19 +4596,19 @@ gen_ipfchostop(compiler_state_t *cstate, const u_char *eaddr, int dir) return b1; case Q_ADDR1: - bpf_error(cstate, "'addr1' is only supported on 802.11"); + bpf_error(cstate, "'addr1' and 'address1' are only supported on 802.11"); break; case Q_ADDR2: - bpf_error(cstate, "'addr2' is only supported on 802.11"); + bpf_error(cstate, "'addr2' and 'address2' are only supported on 802.11"); break; case Q_ADDR3: - bpf_error(cstate, "'addr3' is only supported on 802.11"); + bpf_error(cstate, "'addr3' and 'address3' are only supported on 802.11"); break; case Q_ADDR4: - bpf_error(cstate, "'addr4' is only supported on 802.11"); + bpf_error(cstate, "'addr4' and 'address4' are only supported on 802.11"); break; case Q_RA: @@ -4641,7 +4724,7 @@ gen_mpls_linktype(compiler_state_t *cstate, int proto) case Q_IP: /* match the bottom-of-stack bit */ - b0 = gen_mcmp(cstate, OR_LINKPL, -2, BPF_B, 0x01, 0x01); + b0 = gen_mcmp(cstate, OR_LINKPL, (u_int)-2, BPF_B, 0x01, 0x01); /* match the IPv4 version number */ b1 = gen_mcmp(cstate, OR_LINKPL, 0, BPF_B, 0x40, 0xf0); gen_and(b0, b1); @@ -4649,7 +4732,7 @@ gen_mpls_linktype(compiler_state_t *cstate, int proto) case Q_IPV6: /* match the bottom-of-stack bit */ - b0 = gen_mcmp(cstate, OR_LINKPL, -2, BPF_B, 0x01, 0x01); + b0 = gen_mcmp(cstate, OR_LINKPL, (u_int)-2, BPF_B, 0x01, 0x01); /* match the IPv4 version number */ b1 = gen_mcmp(cstate, OR_LINKPL, 0, BPF_B, 0x60, 0xf0); gen_and(b0, b1); @@ -4909,10 +4992,12 @@ gen_host6(compiler_state_t *cstate, struct in6_addr *addr, #ifndef INET6 static struct block * -gen_gateway(compiler_state_t *cstate, const u_char *eaddr, bpf_u_int32 **alist, - int proto, int dir) +gen_gateway(compiler_state_t *cstate, const u_char *eaddr, + struct addrinfo *alist, int proto, int dir) { struct block *b0, *b1, *tmp; + struct addrinfo *ai; + struct sockaddr_in *sin; if (dir != 0) bpf_error(cstate, "direction applied to 'gateway'"); @@ -4960,12 +5045,48 @@ gen_gateway(compiler_state_t *cstate, const u_char *eaddr, bpf_u_int32 **alist, bpf_error(cstate, "'gateway' supported only on ethernet/FDDI/token ring/802.11/ATM LANE/Fibre Channel"); } - b1 = gen_host(cstate, **alist++, 0xffffffff, proto, Q_OR, Q_HOST); - while (*alist) { - tmp = gen_host(cstate, **alist++, 0xffffffff, proto, Q_OR, - Q_HOST); - gen_or(b1, tmp); - b1 = tmp; + b1 = NULL; + for (ai = alist; ai != NULL; ai = ai->ai_next) { + /* + * Does it have an address? + */ + if (ai->ai_addr != NULL) { + /* + * Yes. Is it an IPv4 address? + */ + if (ai->ai_addr->sa_family == AF_INET) { + /* + * Generate an entry for it. + */ + sin = (struct sockaddr_in *)ai->ai_addr; + tmp = gen_host(cstate, + ntohl(sin->sin_addr.s_addr), + 0xffffffff, proto, Q_OR, Q_HOST); + /* + * Is it the *first* IPv4 address? + */ + if (b1 == NULL) { + /* + * Yes, so start with it. + */ + b1 = tmp; + } else { + /* + * No, so OR it into the + * existing set of + * addresses. + */ + gen_or(b1, tmp); + b1 = tmp; + } + } + } + } + if (b1 == NULL) { + /* + * No IPv4 addresses found. + */ + return (NULL); } gen_not(b1); gen_and(b0, b1); @@ -5647,9 +5768,7 @@ lookup_proto(compiler_state_t *cstate, const char *name, int proto) #if 0 struct stmt * -gen_joinsp(s, n) - struct stmt **s; - int n; +gen_joinsp(struct stmt **s, int n) { return NULL; } @@ -5699,7 +5818,7 @@ gen_protochain(compiler_state_t *cstate, int v, int proto, int dir) if (cstate->off_linkpl.is_variable) bpf_error(cstate, "'protochain' not supported with variable length headers"); - cstate->no_optimize = 1; /*this code is not compatible with optimzer yet */ + cstate->no_optimize = 1; /* this code is not compatible with optimizer yet */ /* * s[0] is a dummy entry to protect other BPF insn from damage @@ -6203,13 +6322,11 @@ gen_scode(compiler_state_t *cstate, const char *name, struct qual q) int tproto; u_char *eaddr; bpf_u_int32 mask, addr; -#ifndef INET6 - bpf_u_int32 **alist; -#else - int tproto6; + struct addrinfo *res, *res0; struct sockaddr_in *sin4; +#ifdef INET6 + int tproto6; struct sockaddr_in6 *sin6; - struct addrinfo *res, *res0; struct in6_addr mask128; #endif /*INET6*/ struct block *b, *tmp; @@ -6308,46 +6425,39 @@ gen_scode(compiler_state_t *cstate, const char *name, struct qual q) */ return (gen_host(cstate, dn_addr, 0, proto, dir, q.addr)); } else { -#ifndef INET6 - alist = pcap_nametoaddr(name); - if (alist == NULL || *alist == NULL) - bpf_error(cstate, "unknown host '%s'", name); - tproto = proto; - if (cstate->off_linktype.constant_part == OFFSET_NOT_SET && - tproto == Q_DEFAULT) - tproto = Q_IP; - b = gen_host(cstate, **alist++, 0xffffffff, tproto, dir, q.addr); - while (*alist) { - tmp = gen_host(cstate, **alist++, 0xffffffff, - tproto, dir, q.addr); - gen_or(b, tmp); - b = tmp; - } - return b; -#else +#ifdef INET6 memset(&mask128, 0xff, sizeof(mask128)); +#endif res0 = res = pcap_nametoaddrinfo(name); if (res == NULL) bpf_error(cstate, "unknown host '%s'", name); cstate->ai = res; b = tmp = NULL; - tproto = tproto6 = proto; + tproto = proto; +#ifdef INET6 + tproto6 = proto; +#endif if (cstate->off_linktype.constant_part == OFFSET_NOT_SET && tproto == Q_DEFAULT) { tproto = Q_IP; +#ifdef INET6 tproto6 = Q_IPV6; +#endif } for (res = res0; res; res = res->ai_next) { switch (res->ai_family) { case AF_INET: +#ifdef INET6 if (tproto == Q_IPV6) continue; +#endif sin4 = (struct sockaddr_in *) res->ai_addr; tmp = gen_host(cstate, ntohl(sin4->sin_addr.s_addr), 0xffffffff, tproto, dir, q.addr); break; +#ifdef INET6 case AF_INET6: if (tproto6 == Q_IP) continue; @@ -6357,6 +6467,7 @@ gen_scode(compiler_state_t *cstate, const char *name, struct qual q) tmp = gen_host6(cstate, &sin6->sin6_addr, &mask128, tproto6, dir, q.addr); break; +#endif default: continue; } @@ -6373,7 +6484,6 @@ gen_scode(compiler_state_t *cstate, const char *name, struct qual q) : " for specified address family"); } return b; -#endif /*INET6*/ } case Q_PORT: @@ -6471,11 +6581,15 @@ gen_scode(compiler_state_t *cstate, const char *name, struct qual q) if (eaddr == NULL) bpf_error(cstate, "unknown ether host: %s", name); - alist = pcap_nametoaddr(name); - if (alist == NULL || *alist == NULL) + res = pcap_nametoaddrinfo(name); + cstate->ai = res; + if (res == NULL) + bpf_error(cstate, "unknown host '%s'", name); + b = gen_gateway(cstate, eaddr, res, proto, dir); + cstate->ai = NULL; + freeaddrinfo(res); + if (b == NULL) bpf_error(cstate, "unknown host '%s'", name); - b = gen_gateway(cstate, eaddr, alist, proto, dir); - free(eaddr); return b; #else bpf_error(cstate, "'gateway' not supported in this configuration"); @@ -6548,7 +6662,6 @@ gen_mcode(compiler_state_t *cstate, const char *s1, const char *s2, /* NOTREACHED */ } /* NOTREACHED */ - return NULL; } struct block * @@ -6667,7 +6780,7 @@ gen_mcode6(compiler_state_t *cstate, const char *s1, const char *s2, struct in6_addr *addr; struct in6_addr mask; struct block *b; - u_int32_t *a, *m; + uint32_t *a, *m; if (s2) bpf_error(cstate, "no mask %s supported", s2); @@ -6689,8 +6802,8 @@ gen_mcode6(compiler_state_t *cstate, const char *s1, const char *s2, (0xff << (8 - masklen % 8)) & 0xff; } - a = (u_int32_t *)addr; - m = (u_int32_t *)&mask; + a = (uint32_t *)addr; + m = (uint32_t *)&mask; if ((a[0] & ~m[0]) || (a[1] & ~m[1]) || (a[2] & ~m[2]) || (a[3] & ~m[3])) { bpf_error(cstate, "non-network bits set in \"%s/%d\"", s1, masklen); @@ -6714,7 +6827,6 @@ gen_mcode6(compiler_state_t *cstate, const char *s1, const char *s2, bpf_error(cstate, "invalid qualifier against IPv6 address"); /* NOTREACHED */ } - return NULL; } #endif /*INET6*/ @@ -6752,12 +6864,10 @@ gen_ecode(compiler_state_t *cstate, const u_char *eaddr, struct qual q) } bpf_error(cstate, "ethernet address used in non-ether expression"); /* NOTREACHED */ - return NULL; } void -sappend(s0, s1) - struct slist *s0, *s1; +sappend(struct slist *s0, struct slist *s1) { /* * This is definitely not the best way to do this, but the @@ -7007,8 +7117,58 @@ gen_load(compiler_state_t *cstate, int proto, struct arth *inst, int size) inst->b = b; break; case Q_ICMPV6: - bpf_error(cstate, "IPv6 upper-layer protocol is not supported by proto[x]"); - /*NOTREACHED*/ + /* + * Do the computation only if the packet contains + * the protocol in question. + */ + b = gen_proto_abbrev(cstate, Q_IPV6); + if (inst->b) { + gen_and(inst->b, b); + } + inst->b = b; + + /* + * Check if we have an icmp6 next header + */ + b = gen_cmp(cstate, OR_LINKPL, 6, BPF_B, 58); + if (inst->b) { + gen_and(inst->b, b); + } + inst->b = b; + + + s = gen_abs_offset_varpart(cstate, &cstate->off_linkpl); + /* + * If "s" is non-null, it has code to arrange that the + * X register contains the variable part of the offset + * of the link-layer payload. Add to it the offset + * computed into the register specified by "index", + * and move that into the X register. Otherwise, just + * load into the X register the offset computed into + * the register specified by "index". + */ + if (s != NULL) { + sappend(s, xfer_to_a(cstate, inst)); + sappend(s, new_stmt(cstate, BPF_ALU|BPF_ADD|BPF_X)); + sappend(s, new_stmt(cstate, BPF_MISC|BPF_TAX)); + } else { + s = xfer_to_x(cstate, inst); + } + + /* + * Load the item at the sum of the offset we've put in the + * X register, the offset of the start of the network + * layer header from the beginning of the link-layer + * payload, and the constant part of the offset of the + * start of the link-layer payload. + */ + tmp = new_stmt(cstate, BPF_LD|BPF_IND|size); + tmp->s.k = cstate->off_linkpl.constant_part + cstate->off_nl + 40; + + sappend(s, tmp); + sappend(inst->s, s); + + break; } inst->regno = regno; s = new_stmt(cstate, BPF_ST); @@ -7180,7 +7340,6 @@ alloc_reg(compiler_state_t *cstate) } bpf_error(cstate, "too many registers needed to evaluate expression"); /* NOTREACHED */ - return 0; } /* @@ -7335,7 +7494,6 @@ gen_broadcast(compiler_state_t *cstate, int proto) } bpf_error(cstate, "only link-layer/IP broadcast filters supported"); /* NOTREACHED */ - return NULL; } /* @@ -7531,7 +7689,6 @@ gen_multicast(compiler_state_t *cstate, int proto) } bpf_error(cstate, "link-layer multicast filters supported only on ethernet/FDDI/token ring/ARCNET/802.11/ATM LANE/Fibre Channel"); /* NOTREACHED */ - return NULL; } /* @@ -7632,8 +7789,18 @@ gen_inbound(compiler_state_t *cstate, int dir) default: /* * If we have packet meta-data indicating a direction, - * check it, otherwise give up as this link-layer type - * has nothing in the packet data. + * and that metadata can be checked by BPF code, check + * it. Otherwise, give up, as this link-layer type has + * nothing in the packet data. + * + * Currently, the only platform where a BPF filter can + * check that metadata is Linux with the in-kernel + * BPF interpreter. If other packet capture mechanisms + * and BPF filters also supported this, it would be + * nice. It would be even better if they made that + * metadata available so that we could provide it + * with newer capture APIs, allowing it to be saved + * in pcapng files. */ #if defined(linux) && defined(PF_PACKET) && defined(SO_ATTACH_FILTER) /* @@ -7659,7 +7826,6 @@ gen_inbound(compiler_state_t *cstate, int dir) #else /* defined(linux) && defined(PF_PACKET) && defined(SO_ATTACH_FILTER) */ bpf_error(cstate, "inbound/outbound not supported on linktype %d", cstate->linktype); - b0 = NULL; /* NOTREACHED */ #endif /* defined(linux) && defined(PF_PACKET) && defined(SO_ATTACH_FILTER) */ } @@ -7776,51 +7942,45 @@ gen_pf_action(compiler_state_t *cstate, int action) } #else /* !HAVE_NET_PFVAR_H */ struct block * -gen_pf_ifname(compiler_state_t *cstate, const char *ifname) +gen_pf_ifname(compiler_state_t *cstate, const char *ifname _U_) { bpf_error(cstate, "libpcap was compiled without pf support"); /* NOTREACHED */ - return (NULL); } struct block * -gen_pf_ruleset(compiler_state_t *cstate, char *ruleset) +gen_pf_ruleset(compiler_state_t *cstate, char *ruleset _U_) { bpf_error(cstate, "libpcap was compiled on a machine without pf support"); /* NOTREACHED */ - return (NULL); } struct block * -gen_pf_rnr(compiler_state_t *cstate, int rnr) +gen_pf_rnr(compiler_state_t *cstate, int rnr _U_) { bpf_error(cstate, "libpcap was compiled on a machine without pf support"); /* NOTREACHED */ - return (NULL); } struct block * -gen_pf_srnr(compiler_state_t *cstate, int srnr) +gen_pf_srnr(compiler_state_t *cstate, int srnr _U_) { bpf_error(cstate, "libpcap was compiled on a machine without pf support"); /* NOTREACHED */ - return (NULL); } struct block * -gen_pf_reason(compiler_state_t *cstate, int reason) +gen_pf_reason(compiler_state_t *cstate, int reason _U_) { bpf_error(cstate, "libpcap was compiled on a machine without pf support"); /* NOTREACHED */ - return (NULL); } struct block * -gen_pf_action(compiler_state_t *cstate, int action) +gen_pf_action(compiler_state_t *cstate, int action _U_) { bpf_error(cstate, "libpcap was compiled on a machine without pf support"); /* NOTREACHED */ - return (NULL); } #endif /* HAVE_NET_PFVAR_H */ @@ -7892,9 +8052,6 @@ gen_acode(compiler_state_t *cstate, const u_char *eaddr, struct qual q) bpf_error(cstate, "aid supported only on ARCnet"); /* NOTREACHED */ } - bpf_error(cstate, "ARCnet address used in non-arc expression"); - /* NOTREACHED */ - return NULL; } static struct block * @@ -7924,19 +8081,19 @@ gen_ahostop(compiler_state_t *cstate, const u_char *eaddr, int dir) return b1; case Q_ADDR1: - bpf_error(cstate, "'addr1' is only supported on 802.11"); + bpf_error(cstate, "'addr1' and 'address1' are only supported on 802.11"); break; case Q_ADDR2: - bpf_error(cstate, "'addr2' is only supported on 802.11"); + bpf_error(cstate, "'addr2' and 'address2' are only supported on 802.11"); break; case Q_ADDR3: - bpf_error(cstate, "'addr3' is only supported on 802.11"); + bpf_error(cstate, "'addr3' and 'address3' are only supported on 802.11"); break; case Q_ADDR4: - bpf_error(cstate, "'addr4' is only supported on 802.11"); + bpf_error(cstate, "'addr4' and 'address4' are only supported on 802.11"); break; case Q_RA: @@ -7951,74 +8108,181 @@ gen_ahostop(compiler_state_t *cstate, const u_char *eaddr, int dir) /* NOTREACHED */ } -#if defined(SKF_AD_VLAN_TAG) && defined(SKF_AD_VLAN_TAG_PRESENT) static struct block * -gen_vlan_bpf_extensions(compiler_state_t *cstate, int vlan_num) +gen_vlan_tpid_test(compiler_state_t *cstate) { - struct block *b0, *b1; - struct slist *s; - - /* generate new filter code based on extracting packet - * metadata */ - s = new_stmt(cstate, BPF_LD|BPF_B|BPF_ABS); - s->s.k = SKF_AD_OFF + SKF_AD_VLAN_TAG_PRESENT; - - b0 = new_block(cstate, JMP(BPF_JEQ)); - b0->stmts = s; - b0->s.k = 1; - - if (vlan_num >= 0) { - s = new_stmt(cstate, BPF_LD|BPF_B|BPF_ABS); - s->s.k = SKF_AD_OFF + SKF_AD_VLAN_TAG; + struct block *b0, *b1; - b1 = new_block(cstate, JMP(BPF_JEQ)); - b1->stmts = s; - b1->s.k = (bpf_int32) vlan_num; + /* check for VLAN, including QinQ */ + b0 = gen_linktype(cstate, ETHERTYPE_8021Q); + b1 = gen_linktype(cstate, ETHERTYPE_8021AD); + gen_or(b0,b1); + b0 = b1; + b1 = gen_linktype(cstate, ETHERTYPE_8021QINQ); + gen_or(b0,b1); - gen_and(b0,b1); - b0 = b1; - } + return b1; +} - return b0; +static struct block * +gen_vlan_vid_test(compiler_state_t *cstate, int vlan_num) +{ + return gen_mcmp(cstate, OR_LINKPL, 0, BPF_H, (bpf_int32)vlan_num, 0x0fff); } -#endif static struct block * gen_vlan_no_bpf_extensions(compiler_state_t *cstate, int vlan_num) { - struct block *b0, *b1; - - /* check for VLAN, including QinQ */ - b0 = gen_linktype(cstate, ETHERTYPE_8021Q); - b1 = gen_linktype(cstate, ETHERTYPE_8021AD); - gen_or(b0,b1); - b0 = b1; - b1 = gen_linktype(cstate, ETHERTYPE_8021QINQ); - gen_or(b0,b1); - b0 = b1; - - /* If a specific VLAN is requested, check VLAN id */ - if (vlan_num >= 0) { - b1 = gen_mcmp(cstate, OR_LINKPL, 0, BPF_H, - (bpf_int32)vlan_num, 0x0fff); - gen_and(b0, b1); - b0 = b1; - } + struct block *b0, *b1; + + b0 = gen_vlan_tpid_test(cstate); + + if (vlan_num >= 0) { + b1 = gen_vlan_vid_test(cstate, vlan_num); + gen_and(b0, b1); + b0 = b1; + } /* - * The payload follows the full header, including the - * VLAN tags, so skip past this VLAN tag. + * Both payload and link header type follow the VLAN tags so that + * both need to be updated. */ - cstate->off_linkpl.constant_part += 4; + cstate->off_linkpl.constant_part += 4; + cstate->off_linktype.constant_part += 4; + + return b0; +} + +#if defined(SKF_AD_VLAN_TAG_PRESENT) +/* add v to variable part of off */ +static void +gen_vlan_vloffset_add(compiler_state_t *cstate, bpf_abs_offset *off, int v, struct slist *s) +{ + struct slist *s2; + + if (!off->is_variable) + off->is_variable = 1; + if (off->reg == -1) + off->reg = alloc_reg(cstate); + + s2 = new_stmt(cstate, BPF_LD|BPF_MEM); + s2->s.k = off->reg; + sappend(s, s2); + s2 = new_stmt(cstate, BPF_ALU|BPF_ADD|BPF_IMM); + s2->s.k = v; + sappend(s, s2); + s2 = new_stmt(cstate, BPF_ST); + s2->s.k = off->reg; + sappend(s, s2); +} + +/* + * patch block b_tpid (VLAN TPID test) to update variable parts of link payload + * and link type offsets first + */ +static void +gen_vlan_patch_tpid_test(compiler_state_t *cstate, struct block *b_tpid) +{ + struct slist s; + + /* offset determined at run time, shift variable part */ + s.next = NULL; + cstate->is_vlan_vloffset = 1; + gen_vlan_vloffset_add(cstate, &cstate->off_linkpl, 4, &s); + gen_vlan_vloffset_add(cstate, &cstate->off_linktype, 4, &s); + + /* we get a pointer to a chain of or-ed blocks, patch first of them */ + sappend(s.next, b_tpid->head->stmts); + b_tpid->head->stmts = s.next; +} + +/* + * patch block b_vid (VLAN id test) to load VID value either from packet + * metadata (using BPF extensions) if SKF_AD_VLAN_TAG_PRESENT is true + */ +static void +gen_vlan_patch_vid_test(compiler_state_t *cstate, struct block *b_vid) +{ + struct slist *s, *s2, *sjeq; + unsigned cnt; + + s = new_stmt(cstate, BPF_LD|BPF_B|BPF_ABS); + s->s.k = SKF_AD_OFF + SKF_AD_VLAN_TAG_PRESENT; + + /* true -> next instructions, false -> beginning of b_vid */ + sjeq = new_stmt(cstate, JMP(BPF_JEQ)); + sjeq->s.k = 1; + sjeq->s.jf = b_vid->stmts; + sappend(s, sjeq); + + s2 = new_stmt(cstate, BPF_LD|BPF_B|BPF_ABS); + s2->s.k = SKF_AD_OFF + SKF_AD_VLAN_TAG; + sappend(s, s2); + sjeq->s.jt = s2; + + /* jump to the test in b_vid (bypass loading VID from packet data) */ + cnt = 0; + for (s2 = b_vid->stmts; s2; s2 = s2->next) + cnt++; + s2 = new_stmt(cstate, JMP(BPF_JA)); + s2->s.k = cnt; + sappend(s, s2); + + /* insert our statements at the beginning of b_vid */ + sappend(s, b_vid->stmts); + b_vid->stmts = s; +} + +/* + * Generate check for "vlan" or "vlan <id>" on systems with support for BPF + * extensions. Even if kernel supports VLAN BPF extensions, (outermost) VLAN + * tag can be either in metadata or in packet data; therefore if the + * SKF_AD_VLAN_TAG_PRESENT test is negative, we need to check link + * header for VLAN tag. As the decision is done at run time, we need + * update variable part of the offsets + */ +static struct block * +gen_vlan_bpf_extensions(compiler_state_t *cstate, int vlan_num) +{ + struct block *b0, *b_tpid, *b_vid = NULL; + struct slist *s; + + /* generate new filter code based on extracting packet + * metadata */ + s = new_stmt(cstate, BPF_LD|BPF_B|BPF_ABS); + s->s.k = SKF_AD_OFF + SKF_AD_VLAN_TAG_PRESENT; + + b0 = new_block(cstate, JMP(BPF_JEQ)); + b0->stmts = s; + b0->s.k = 1; /* - * The link-layer type information follows the VLAN tags, so - * skip past this VLAN tag. + * This is tricky. We need to insert the statements updating variable + * parts of offsets before the the traditional TPID and VID tests so + * that they are called whenever SKF_AD_VLAN_TAG_PRESENT fails but + * we do not want this update to affect those checks. That's why we + * generate both test blocks first and insert the statements updating + * variable parts of both offsets after that. This wouldn't work if + * there already were variable length link header when entering this + * function but gen_vlan_bpf_extensions() isn't called in that case. */ - cstate->off_linktype.constant_part += 4; + b_tpid = gen_vlan_tpid_test(cstate); + if (vlan_num >= 0) + b_vid = gen_vlan_vid_test(cstate, vlan_num); + + gen_vlan_patch_tpid_test(cstate, b_tpid); + gen_or(b0, b_tpid); + b0 = b_tpid; + + if (vlan_num >= 0) { + gen_vlan_patch_vid_test(cstate, b_vid); + gen_and(b0, b_vid); + b0 = b_vid; + } return b0; } +#endif /* * support IEEE 802.1Q VLAN trunk over ethernet @@ -8068,7 +8332,7 @@ gen_vlan(compiler_state_t *cstate, int vlan_num) case DLT_EN10MB: case DLT_NETANALYZER: case DLT_NETANALYZER_TRANSPARENT: -#if defined(SKF_AD_VLAN_TAG) && defined(SKF_AD_VLAN_TAG_PRESENT) +#if defined(SKF_AD_VLAN_TAG_PRESENT) /* Verify that this is the outer part of the packet and * not encapsulated somehow. */ if (cstate->vlan_stack_depth == 0 && !cstate->off_linkhdr.is_variable && @@ -8140,7 +8404,6 @@ gen_mpls(compiler_state_t *cstate, int label_num) default: bpf_error(cstate, "no MPLS support for data link type %d", cstate->linktype); - b0 = NULL; /*NOTREACHED*/ break; } @@ -8561,7 +8824,7 @@ gen_atmfield_code(compiler_state_t *cstate, int atmfield, bpf_int32 jvalue, case A_VPI: if (!cstate->is_atm) bpf_error(cstate, "'vpi' supported only on raw ATM"); - if (cstate->off_vpi == (u_int)-1) + if (cstate->off_vpi == OFFSET_NOT_SET) abort(); b0 = gen_ncmp(cstate, OR_LINKHDR, cstate->off_vpi, BPF_B, 0xffffffff, jtype, reverse, jvalue); @@ -8570,21 +8833,21 @@ gen_atmfield_code(compiler_state_t *cstate, int atmfield, bpf_int32 jvalue, case A_VCI: if (!cstate->is_atm) bpf_error(cstate, "'vci' supported only on raw ATM"); - if (cstate->off_vci == (u_int)-1) + if (cstate->off_vci == OFFSET_NOT_SET) abort(); b0 = gen_ncmp(cstate, OR_LINKHDR, cstate->off_vci, BPF_H, 0xffffffff, jtype, reverse, jvalue); break; case A_PROTOTYPE: - if (cstate->off_proto == (u_int)-1) + if (cstate->off_proto == OFFSET_NOT_SET) abort(); /* XXX - this isn't on FreeBSD */ b0 = gen_ncmp(cstate, OR_LINKHDR, cstate->off_proto, BPF_B, 0x0f, jtype, reverse, jvalue); break; case A_MSGTYPE: - if (cstate->off_payload == (u_int)-1) + if (cstate->off_payload == OFFSET_NOT_SET) abort(); b0 = gen_ncmp(cstate, OR_LINKHDR, cstate->off_payload + MSG_TYPE_POS, BPF_B, 0xffffffff, jtype, reverse, jvalue); @@ -8593,7 +8856,7 @@ gen_atmfield_code(compiler_state_t *cstate, int atmfield, bpf_int32 jvalue, case A_CALLREFTYPE: if (!cstate->is_atm) bpf_error(cstate, "'callref' supported only on raw ATM"); - if (cstate->off_proto == (u_int)-1) + if (cstate->off_proto == OFFSET_NOT_SET) abort(); b0 = gen_ncmp(cstate, OR_LINKHDR, cstate->off_proto, BPF_B, 0xffffffff, jtype, reverse, jvalue); @@ -8795,7 +9058,7 @@ gen_mtp3field_code(compiler_state_t *cstate, int mtp3field, bpf_u_int32 jvalue, /* FALLTHROUGH */ case M_SIO: - if (cstate->off_sio == (u_int)-1) + if (cstate->off_sio == OFFSET_NOT_SET) bpf_error(cstate, "'sio' supported only on SS7"); /* sio coded on 1 byte so max value 255 */ if(jvalue > 255) @@ -8808,7 +9071,7 @@ gen_mtp3field_code(compiler_state_t *cstate, int mtp3field, bpf_u_int32 jvalue, case MH_OPC: newoff_opc+=3; case M_OPC: - if (cstate->off_opc == (u_int)-1) + if (cstate->off_opc == OFFSET_NOT_SET) bpf_error(cstate, "'opc' supported only on SS7"); /* opc coded on 14 bits so max value 16383 */ if (jvalue > 16383) @@ -8832,7 +9095,7 @@ gen_mtp3field_code(compiler_state_t *cstate, int mtp3field, bpf_u_int32 jvalue, /* FALLTHROUGH */ case M_DPC: - if (cstate->off_dpc == (u_int)-1) + if (cstate->off_dpc == OFFSET_NOT_SET) bpf_error(cstate, "'dpc' supported only on SS7"); /* dpc coded on 14 bits so max value 16383 */ if (jvalue > 16383) @@ -8852,7 +9115,7 @@ gen_mtp3field_code(compiler_state_t *cstate, int mtp3field, bpf_u_int32 jvalue, case MH_SLS: newoff_sls+=3; case M_SLS: - if (cstate->off_sls == (u_int)-1) + if (cstate->off_sls == OFFSET_NOT_SET) bpf_error(cstate, "'sls' supported only on SS7"); /* sls coded on 4 bits so max value 15 */ if (jvalue > 15) diff --git a/freebsd/contrib/libpcap/gencode.h b/freebsd/contrib/libpcap/gencode.h index 2b089d21..88def5a8 100644 --- a/freebsd/contrib/libpcap/gencode.h +++ b/freebsd/contrib/libpcap/gencode.h @@ -19,6 +19,8 @@ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. */ +#include "pcap/funcattrs.h" + /* * ATM support: * @@ -53,10 +55,6 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#ifndef HAVE___ATTRIBUTE__ -#define __attribute__(x) -#endif /* HAVE___ATTRIBUTE__ */ - /* Address qualifiers. */ #define Q_HOST 1 @@ -268,6 +266,11 @@ struct block { int val[N_ATOMS]; }; +/* + * A value of 0 for val[i] means the value is unknown. + */ +#define VAL_UNKNOWN 0 + struct arth { struct block *b; /* protocol checks */ struct slist *s; /* stmt list */ @@ -340,11 +343,29 @@ struct block *gen_mtp2type_abbrev(compiler_state_t *, int type); struct block *gen_mtp3field_code(compiler_state_t *, int, bpf_u_int32, bpf_u_int32, int); +#ifndef HAVE_NET_PFVAR_H +PCAP_NORETURN +#endif struct block *gen_pf_ifname(compiler_state_t *, const char *); +#ifndef HAVE_NET_PFVAR_H +PCAP_NORETURN +#endif struct block *gen_pf_rnr(compiler_state_t *, int); +#ifndef HAVE_NET_PFVAR_H +PCAP_NORETURN +#endif struct block *gen_pf_srnr(compiler_state_t *, int); +#ifndef HAVE_NET_PFVAR_H +PCAP_NORETURN +#endif struct block *gen_pf_ruleset(compiler_state_t *, char *); +#ifndef HAVE_NET_PFVAR_H +PCAP_NORETURN +#endif struct block *gen_pf_reason(compiler_state_t *, int); +#ifndef HAVE_NET_PFVAR_H +PCAP_NORETURN +#endif struct block *gen_pf_action(compiler_state_t *, int); struct block *gen_p80211_type(compiler_state_t *, int, int); @@ -366,20 +387,13 @@ struct icode { }; void bpf_optimize(compiler_state_t *, struct icode *ic); -void bpf_syntax_error(compiler_state_t *, const char *); -void bpf_error(compiler_state_t *, const char *, ...) - __attribute__((noreturn)) -#ifdef __ATTRIBUTE___FORMAT_OK - __attribute__((format (printf, 2, 3))) -#endif /* __ATTRIBUTE___FORMAT_OK */ - ; +void PCAP_NORETURN bpf_syntax_error(compiler_state_t *, const char *); +void PCAP_NORETURN bpf_error(compiler_state_t *, const char *, ...) + PCAP_PRINTFLIKE(2, 3); void finish_parse(compiler_state_t *, struct block *); char *sdup(compiler_state_t *, const char *); -struct _opt_state; -typedef struct _opt_state opt_state_t; - struct bpf_insn *icode_to_fcode(compiler_state_t *, struct icode *, struct block *, u_int *); void sappend(struct slist *, struct slist *); diff --git a/freebsd/contrib/libpcap/grammar.c b/freebsd/contrib/libpcap/grammar.c index a04f770c..4672c0b6 100644 --- a/freebsd/contrib/libpcap/grammar.c +++ b/freebsd/contrib/libpcap/grammar.c @@ -118,23 +118,18 @@ * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. * - * $FreeBSD$ */ #ifdef HAVE_CONFIG_H -#include "config.h" +#include <config.h> #endif -#ifdef _WIN32 -#include <pcap-stdinc.h> -#else /* _WIN32 */ -#include <sys/types.h> -#include <sys/socket.h> -#endif /* _WIN32 */ - #include <stdlib.h> #ifndef _WIN32 +#include <sys/types.h> +#include <sys/socket.h> + #if __STDC__ struct mbuf; struct rtentry; @@ -146,6 +141,8 @@ struct rtentry; #include <stdio.h> +#include "diag-control.h" + #include "pcap-int.h" #include "gencode.h" @@ -154,7 +151,7 @@ struct rtentry; #ifdef HAVE_NET_PFVAR_H #include <net/if.h> -#include <netpfil/pf/pf.h> +#include <net/pfvar.h> #include <net/if_pflog.h> #endif #include "llc.h" @@ -165,9 +162,30 @@ struct rtentry; #include "os-proto.h" #endif -#define QSET(q, p, d, a) (q).proto = (p),\ - (q).dir = (d),\ - (q).addr = (a) +#ifdef YYBYACC +/* + * Both Berkeley YACC and Bison define yydebug (under whatever name + * it has) as a global, but Bison does so only if YYDEBUG is defined. + * Berkeley YACC define it even if YYDEBUG isn't defined; declare it + * here to suppress a warning. + */ +#if !defined(YYDEBUG) +extern int yydebug; +#endif + +/* + * In Berkeley YACC, yynerrs (under whatever name it has) is global, + * even if it's building a reentrant parser. In Bison, it's local + * in reentrant parsers. + * + * Declare it to squelch a warning. + */ +extern int yynerrs; +#endif + +#define QSET(q, p, d, a) (q).proto = (unsigned char)(p),\ + (q).dir = (unsigned char)(d),\ + (q).addr = (unsigned char)(a) struct tok { int v; /* value */ @@ -274,8 +292,8 @@ str2tok(const char *str, const struct tok *toks) static struct qual qerr = { Q_UNDEF, Q_UNDEF, Q_UNDEF, Q_UNDEF }; -static void -yyerror(void *yyscanner, compiler_state_t *cstate, const char *msg) +static PCAP_NORETURN_DEF void +yyerror(void *yyscanner _U_, compiler_state_t *cstate, const char *msg) { bpf_syntax_error(cstate, msg); /* NOTREACHED */ @@ -321,33 +339,29 @@ pfaction_to_num(compiler_state_t *cstate, const char *action) } } #else /* !HAVE_NET_PFVAR_H */ -static int -pfreason_to_num(compiler_state_t *cstate, const char *reason) +static PCAP_NORETURN_DEF int +pfreason_to_num(compiler_state_t *cstate, const char *reason _U_) { bpf_error(cstate, "libpcap was compiled on a machine without pf support"); /*NOTREACHED*/ - - /* this is to make the VC compiler happy */ - return -1; } -static int -pfaction_to_num(compiler_state_t *cstate, const char *action) +static PCAP_NORETURN_DEF int +pfaction_to_num(compiler_state_t *cstate, const char *action _U_) { bpf_error(cstate, "libpcap was compiled on a machine without pf support"); /*NOTREACHED*/ - - /* this is to make the VC compiler happy */ - return -1; } #endif /* HAVE_NET_PFVAR_H */ + +DIAG_OFF_BISON_BYACC #ifdef YYSTYPE #undef YYSTYPE_IS_DECLARED #define YYSTYPE_IS_DECLARED 1 #endif #ifndef YYSTYPE_IS_DECLARED #define YYSTYPE_IS_DECLARED 1 -#line 272 "grammar.y" +#line 286 "grammar.y" typedef union { int i; bpf_u_int32 h; @@ -364,7 +378,7 @@ typedef union { struct block *rblk; } YYSTYPE; #endif /* !YYSTYPE_IS_DECLARED */ -#line 368 "grammar.c" +#line 382 "grammar.c" /* compatibility with bison */ #ifdef YYPARSE_PARAM @@ -1434,64 +1448,64 @@ yyreduce: switch (yyn) { case 1: -#line 347 "grammar.y" +#line 361 "grammar.y" { finish_parse(cstate, yystack.l_mark[0].blk.b); } break; case 3: -#line 352 "grammar.y" +#line 366 "grammar.y" { yyval.blk.q = qerr; } break; case 5: -#line 355 "grammar.y" +#line 369 "grammar.y" { gen_and(yystack.l_mark[-2].blk.b, yystack.l_mark[0].blk.b); yyval.blk = yystack.l_mark[0].blk; } break; case 6: -#line 356 "grammar.y" +#line 370 "grammar.y" { gen_and(yystack.l_mark[-2].blk.b, yystack.l_mark[0].blk.b); yyval.blk = yystack.l_mark[0].blk; } break; case 7: -#line 357 "grammar.y" +#line 371 "grammar.y" { gen_or(yystack.l_mark[-2].blk.b, yystack.l_mark[0].blk.b); yyval.blk = yystack.l_mark[0].blk; } break; case 8: -#line 358 "grammar.y" +#line 372 "grammar.y" { gen_or(yystack.l_mark[-2].blk.b, yystack.l_mark[0].blk.b); yyval.blk = yystack.l_mark[0].blk; } break; case 9: -#line 360 "grammar.y" +#line 374 "grammar.y" { yyval.blk = yystack.l_mark[-1].blk; } break; case 10: -#line 362 "grammar.y" +#line 376 "grammar.y" { yyval.blk = yystack.l_mark[-1].blk; } break; case 12: -#line 365 "grammar.y" +#line 379 "grammar.y" { yyval.blk.b = gen_ncode(cstate, NULL, (bpf_u_int32)yystack.l_mark[0].i, yyval.blk.q = yystack.l_mark[-1].blk.q); } break; case 13: -#line 367 "grammar.y" +#line 381 "grammar.y" { yyval.blk = yystack.l_mark[-1].blk; } break; case 14: -#line 369 "grammar.y" +#line 383 "grammar.y" { yyval.blk.b = gen_scode(cstate, yystack.l_mark[0].s, yyval.blk.q = yystack.l_mark[-1].blk.q); } break; case 15: -#line 370 "grammar.y" +#line 384 "grammar.y" { yyval.blk.b = gen_mcode(cstate, yystack.l_mark[-2].s, NULL, yystack.l_mark[0].i, yyval.blk.q = yystack.l_mark[-3].blk.q); } break; case 16: -#line 372 "grammar.y" +#line 386 "grammar.y" { yyval.blk.b = gen_mcode(cstate, yystack.l_mark[-2].s, yystack.l_mark[0].s, 0, yyval.blk.q = yystack.l_mark[-3].blk.q); } break; case 17: -#line 374 "grammar.y" +#line 388 "grammar.y" { /* Decide how to parse HID based on proto */ yyval.blk.q = yystack.l_mark[-1].blk.q; @@ -1507,7 +1521,7 @@ case 17: } break; case 18: -#line 387 "grammar.y" +#line 401 "grammar.y" { #ifdef INET6 yyval.blk.b = gen_mcode6(cstate, yystack.l_mark[-2].s, NULL, yystack.l_mark[0].i, @@ -1519,7 +1533,7 @@ case 18: } break; case 19: -#line 396 "grammar.y" +#line 410 "grammar.y" { #ifdef INET6 yyval.blk.b = gen_mcode6(cstate, yystack.l_mark[0].s, 0, 128, @@ -1531,7 +1545,7 @@ case 19: } break; case 20: -#line 405 "grammar.y" +#line 419 "grammar.y" { yyval.blk.b = gen_ecode(cstate, yystack.l_mark[0].e, yyval.blk.q = yystack.l_mark[-1].blk.q); /* @@ -1543,7 +1557,7 @@ case 20: } break; case 21: -#line 414 "grammar.y" +#line 428 "grammar.y" { yyval.blk.b = gen_acode(cstate, yystack.l_mark[0].e, yyval.blk.q = yystack.l_mark[-1].blk.q); /* @@ -1555,465 +1569,465 @@ case 21: } break; case 22: -#line 423 "grammar.y" +#line 437 "grammar.y" { gen_not(yystack.l_mark[0].blk.b); yyval.blk = yystack.l_mark[0].blk; } break; case 23: -#line 425 "grammar.y" +#line 439 "grammar.y" { yyval.blk = yystack.l_mark[-1].blk; } break; case 24: -#line 427 "grammar.y" +#line 441 "grammar.y" { yyval.blk = yystack.l_mark[-1].blk; } break; case 26: -#line 430 "grammar.y" +#line 444 "grammar.y" { gen_and(yystack.l_mark[-2].blk.b, yystack.l_mark[0].blk.b); yyval.blk = yystack.l_mark[0].blk; } break; case 27: -#line 431 "grammar.y" +#line 445 "grammar.y" { gen_or(yystack.l_mark[-2].blk.b, yystack.l_mark[0].blk.b); yyval.blk = yystack.l_mark[0].blk; } break; case 28: -#line 433 "grammar.y" +#line 447 "grammar.y" { yyval.blk.b = gen_ncode(cstate, NULL, (bpf_u_int32)yystack.l_mark[0].i, yyval.blk.q = yystack.l_mark[-1].blk.q); } break; case 31: -#line 438 "grammar.y" +#line 452 "grammar.y" { gen_not(yystack.l_mark[0].blk.b); yyval.blk = yystack.l_mark[0].blk; } break; case 32: -#line 440 "grammar.y" +#line 454 "grammar.y" { QSET(yyval.blk.q, yystack.l_mark[-2].i, yystack.l_mark[-1].i, yystack.l_mark[0].i); } break; case 33: -#line 441 "grammar.y" +#line 455 "grammar.y" { QSET(yyval.blk.q, yystack.l_mark[-1].i, yystack.l_mark[0].i, Q_DEFAULT); } break; case 34: -#line 442 "grammar.y" +#line 456 "grammar.y" { QSET(yyval.blk.q, yystack.l_mark[-1].i, Q_DEFAULT, yystack.l_mark[0].i); } break; case 35: -#line 443 "grammar.y" +#line 457 "grammar.y" { QSET(yyval.blk.q, yystack.l_mark[-1].i, Q_DEFAULT, Q_PROTO); } break; case 36: -#line 444 "grammar.y" +#line 458 "grammar.y" { QSET(yyval.blk.q, yystack.l_mark[-1].i, Q_DEFAULT, Q_PROTOCHAIN); } break; case 37: -#line 445 "grammar.y" +#line 459 "grammar.y" { QSET(yyval.blk.q, yystack.l_mark[-1].i, Q_DEFAULT, yystack.l_mark[0].i); } break; case 38: -#line 447 "grammar.y" +#line 461 "grammar.y" { yyval.blk = yystack.l_mark[0].blk; } break; case 39: -#line 448 "grammar.y" +#line 462 "grammar.y" { yyval.blk.b = yystack.l_mark[-1].blk.b; yyval.blk.q = yystack.l_mark[-2].blk.q; } break; case 40: -#line 449 "grammar.y" +#line 463 "grammar.y" { yyval.blk.b = gen_proto_abbrev(cstate, yystack.l_mark[0].i); yyval.blk.q = qerr; } break; case 41: -#line 450 "grammar.y" +#line 464 "grammar.y" { yyval.blk.b = gen_relation(cstate, yystack.l_mark[-1].i, yystack.l_mark[-2].a, yystack.l_mark[0].a, 0); yyval.blk.q = qerr; } break; case 42: -#line 452 "grammar.y" +#line 466 "grammar.y" { yyval.blk.b = gen_relation(cstate, yystack.l_mark[-1].i, yystack.l_mark[-2].a, yystack.l_mark[0].a, 1); yyval.blk.q = qerr; } break; case 43: -#line 454 "grammar.y" +#line 468 "grammar.y" { yyval.blk.b = yystack.l_mark[0].rblk; yyval.blk.q = qerr; } break; case 44: -#line 455 "grammar.y" +#line 469 "grammar.y" { yyval.blk.b = gen_atmtype_abbrev(cstate, yystack.l_mark[0].i); yyval.blk.q = qerr; } break; case 45: -#line 456 "grammar.y" +#line 470 "grammar.y" { yyval.blk.b = gen_atmmulti_abbrev(cstate, yystack.l_mark[0].i); yyval.blk.q = qerr; } break; case 46: -#line 457 "grammar.y" +#line 471 "grammar.y" { yyval.blk.b = yystack.l_mark[0].blk.b; yyval.blk.q = qerr; } break; case 47: -#line 458 "grammar.y" +#line 472 "grammar.y" { yyval.blk.b = gen_mtp2type_abbrev(cstate, yystack.l_mark[0].i); yyval.blk.q = qerr; } break; case 48: -#line 459 "grammar.y" +#line 473 "grammar.y" { yyval.blk.b = yystack.l_mark[0].blk.b; yyval.blk.q = qerr; } break; case 50: -#line 463 "grammar.y" +#line 477 "grammar.y" { yyval.i = Q_DEFAULT; } break; case 51: -#line 466 "grammar.y" +#line 480 "grammar.y" { yyval.i = Q_SRC; } break; case 52: -#line 467 "grammar.y" +#line 481 "grammar.y" { yyval.i = Q_DST; } break; case 53: -#line 468 "grammar.y" +#line 482 "grammar.y" { yyval.i = Q_OR; } break; case 54: -#line 469 "grammar.y" +#line 483 "grammar.y" { yyval.i = Q_OR; } break; case 55: -#line 470 "grammar.y" +#line 484 "grammar.y" { yyval.i = Q_AND; } break; case 56: -#line 471 "grammar.y" +#line 485 "grammar.y" { yyval.i = Q_AND; } break; case 57: -#line 472 "grammar.y" +#line 486 "grammar.y" { yyval.i = Q_ADDR1; } break; case 58: -#line 473 "grammar.y" +#line 487 "grammar.y" { yyval.i = Q_ADDR2; } break; case 59: -#line 474 "grammar.y" +#line 488 "grammar.y" { yyval.i = Q_ADDR3; } break; case 60: -#line 475 "grammar.y" +#line 489 "grammar.y" { yyval.i = Q_ADDR4; } break; case 61: -#line 476 "grammar.y" +#line 490 "grammar.y" { yyval.i = Q_RA; } break; case 62: -#line 477 "grammar.y" +#line 491 "grammar.y" { yyval.i = Q_TA; } break; case 63: -#line 480 "grammar.y" +#line 494 "grammar.y" { yyval.i = Q_HOST; } break; case 64: -#line 481 "grammar.y" +#line 495 "grammar.y" { yyval.i = Q_NET; } break; case 65: -#line 482 "grammar.y" +#line 496 "grammar.y" { yyval.i = Q_PORT; } break; case 66: -#line 483 "grammar.y" +#line 497 "grammar.y" { yyval.i = Q_PORTRANGE; } break; case 67: -#line 486 "grammar.y" +#line 500 "grammar.y" { yyval.i = Q_GATEWAY; } break; case 68: -#line 488 "grammar.y" +#line 502 "grammar.y" { yyval.i = Q_LINK; } break; case 69: -#line 489 "grammar.y" +#line 503 "grammar.y" { yyval.i = Q_IP; } break; case 70: -#line 490 "grammar.y" +#line 504 "grammar.y" { yyval.i = Q_ARP; } break; case 71: -#line 491 "grammar.y" +#line 505 "grammar.y" { yyval.i = Q_RARP; } break; case 72: -#line 492 "grammar.y" +#line 506 "grammar.y" { yyval.i = Q_SCTP; } break; case 73: -#line 493 "grammar.y" +#line 507 "grammar.y" { yyval.i = Q_TCP; } break; case 74: -#line 494 "grammar.y" +#line 508 "grammar.y" { yyval.i = Q_UDP; } break; case 75: -#line 495 "grammar.y" +#line 509 "grammar.y" { yyval.i = Q_ICMP; } break; case 76: -#line 496 "grammar.y" +#line 510 "grammar.y" { yyval.i = Q_IGMP; } break; case 77: -#line 497 "grammar.y" +#line 511 "grammar.y" { yyval.i = Q_IGRP; } break; case 78: -#line 498 "grammar.y" +#line 512 "grammar.y" { yyval.i = Q_PIM; } break; case 79: -#line 499 "grammar.y" +#line 513 "grammar.y" { yyval.i = Q_VRRP; } break; case 80: -#line 500 "grammar.y" +#line 514 "grammar.y" { yyval.i = Q_CARP; } break; case 81: -#line 501 "grammar.y" +#line 515 "grammar.y" { yyval.i = Q_ATALK; } break; case 82: -#line 502 "grammar.y" +#line 516 "grammar.y" { yyval.i = Q_AARP; } break; case 83: -#line 503 "grammar.y" +#line 517 "grammar.y" { yyval.i = Q_DECNET; } break; case 84: -#line 504 "grammar.y" +#line 518 "grammar.y" { yyval.i = Q_LAT; } break; case 85: -#line 505 "grammar.y" +#line 519 "grammar.y" { yyval.i = Q_SCA; } break; case 86: -#line 506 "grammar.y" +#line 520 "grammar.y" { yyval.i = Q_MOPDL; } break; case 87: -#line 507 "grammar.y" +#line 521 "grammar.y" { yyval.i = Q_MOPRC; } break; case 88: -#line 508 "grammar.y" +#line 522 "grammar.y" { yyval.i = Q_IPV6; } break; case 89: -#line 509 "grammar.y" +#line 523 "grammar.y" { yyval.i = Q_ICMPV6; } break; case 90: -#line 510 "grammar.y" +#line 524 "grammar.y" { yyval.i = Q_AH; } break; case 91: -#line 511 "grammar.y" +#line 525 "grammar.y" { yyval.i = Q_ESP; } break; case 92: -#line 512 "grammar.y" +#line 526 "grammar.y" { yyval.i = Q_ISO; } break; case 93: -#line 513 "grammar.y" +#line 527 "grammar.y" { yyval.i = Q_ESIS; } break; case 94: -#line 514 "grammar.y" +#line 528 "grammar.y" { yyval.i = Q_ISIS; } break; case 95: -#line 515 "grammar.y" +#line 529 "grammar.y" { yyval.i = Q_ISIS_L1; } break; case 96: -#line 516 "grammar.y" +#line 530 "grammar.y" { yyval.i = Q_ISIS_L2; } break; case 97: -#line 517 "grammar.y" +#line 531 "grammar.y" { yyval.i = Q_ISIS_IIH; } break; case 98: -#line 518 "grammar.y" +#line 532 "grammar.y" { yyval.i = Q_ISIS_LSP; } break; case 99: -#line 519 "grammar.y" +#line 533 "grammar.y" { yyval.i = Q_ISIS_SNP; } break; case 100: -#line 520 "grammar.y" +#line 534 "grammar.y" { yyval.i = Q_ISIS_PSNP; } break; case 101: -#line 521 "grammar.y" +#line 535 "grammar.y" { yyval.i = Q_ISIS_CSNP; } break; case 102: -#line 522 "grammar.y" +#line 536 "grammar.y" { yyval.i = Q_CLNP; } break; case 103: -#line 523 "grammar.y" +#line 537 "grammar.y" { yyval.i = Q_STP; } break; case 104: -#line 524 "grammar.y" +#line 538 "grammar.y" { yyval.i = Q_IPX; } break; case 105: -#line 525 "grammar.y" +#line 539 "grammar.y" { yyval.i = Q_NETBEUI; } break; case 106: -#line 526 "grammar.y" +#line 540 "grammar.y" { yyval.i = Q_RADIO; } break; case 107: -#line 528 "grammar.y" +#line 542 "grammar.y" { yyval.rblk = gen_broadcast(cstate, yystack.l_mark[-1].i); } break; case 108: -#line 529 "grammar.y" +#line 543 "grammar.y" { yyval.rblk = gen_multicast(cstate, yystack.l_mark[-1].i); } break; case 109: -#line 530 "grammar.y" +#line 544 "grammar.y" { yyval.rblk = gen_less(cstate, yystack.l_mark[0].i); } break; case 110: -#line 531 "grammar.y" +#line 545 "grammar.y" { yyval.rblk = gen_greater(cstate, yystack.l_mark[0].i); } break; case 111: -#line 532 "grammar.y" +#line 546 "grammar.y" { yyval.rblk = gen_byteop(cstate, yystack.l_mark[-1].i, yystack.l_mark[-2].i, yystack.l_mark[0].i); } break; case 112: -#line 533 "grammar.y" +#line 547 "grammar.y" { yyval.rblk = gen_inbound(cstate, 0); } break; case 113: -#line 534 "grammar.y" +#line 548 "grammar.y" { yyval.rblk = gen_inbound(cstate, 1); } break; case 114: -#line 535 "grammar.y" +#line 549 "grammar.y" { yyval.rblk = gen_vlan(cstate, yystack.l_mark[0].i); } break; case 115: -#line 536 "grammar.y" +#line 550 "grammar.y" { yyval.rblk = gen_vlan(cstate, -1); } break; case 116: -#line 537 "grammar.y" +#line 551 "grammar.y" { yyval.rblk = gen_mpls(cstate, yystack.l_mark[0].i); } break; case 117: -#line 538 "grammar.y" +#line 552 "grammar.y" { yyval.rblk = gen_mpls(cstate, -1); } break; case 118: -#line 539 "grammar.y" +#line 553 "grammar.y" { yyval.rblk = gen_pppoed(cstate); } break; case 119: -#line 540 "grammar.y" +#line 554 "grammar.y" { yyval.rblk = gen_pppoes(cstate, yystack.l_mark[0].i); } break; case 120: -#line 541 "grammar.y" +#line 555 "grammar.y" { yyval.rblk = gen_pppoes(cstate, -1); } break; case 121: -#line 542 "grammar.y" +#line 556 "grammar.y" { yyval.rblk = gen_geneve(cstate, yystack.l_mark[0].i); } break; case 122: -#line 543 "grammar.y" +#line 557 "grammar.y" { yyval.rblk = gen_geneve(cstate, -1); } break; case 123: -#line 544 "grammar.y" +#line 558 "grammar.y" { yyval.rblk = yystack.l_mark[0].rblk; } break; case 124: -#line 545 "grammar.y" +#line 559 "grammar.y" { yyval.rblk = yystack.l_mark[0].rblk; } break; case 125: -#line 546 "grammar.y" +#line 560 "grammar.y" { yyval.rblk = yystack.l_mark[0].rblk; } break; case 126: -#line 549 "grammar.y" +#line 563 "grammar.y" { yyval.rblk = gen_pf_ifname(cstate, yystack.l_mark[0].s); } break; case 127: -#line 550 "grammar.y" +#line 564 "grammar.y" { yyval.rblk = gen_pf_ruleset(cstate, yystack.l_mark[0].s); } break; case 128: -#line 551 "grammar.y" +#line 565 "grammar.y" { yyval.rblk = gen_pf_rnr(cstate, yystack.l_mark[0].i); } break; case 129: -#line 552 "grammar.y" +#line 566 "grammar.y" { yyval.rblk = gen_pf_srnr(cstate, yystack.l_mark[0].i); } break; case 130: -#line 553 "grammar.y" +#line 567 "grammar.y" { yyval.rblk = gen_pf_reason(cstate, yystack.l_mark[0].i); } break; case 131: -#line 554 "grammar.y" +#line 568 "grammar.y" { yyval.rblk = gen_pf_action(cstate, yystack.l_mark[0].i); } break; case 132: -#line 558 "grammar.y" +#line 572 "grammar.y" { yyval.rblk = gen_p80211_type(cstate, yystack.l_mark[-2].i | yystack.l_mark[0].i, IEEE80211_FC0_TYPE_MASK | IEEE80211_FC0_SUBTYPE_MASK); } break; case 133: -#line 562 "grammar.y" +#line 576 "grammar.y" { yyval.rblk = gen_p80211_type(cstate, yystack.l_mark[0].i, IEEE80211_FC0_TYPE_MASK); } break; case 134: -#line 565 "grammar.y" +#line 579 "grammar.y" { yyval.rblk = gen_p80211_type(cstate, yystack.l_mark[0].i, IEEE80211_FC0_TYPE_MASK | IEEE80211_FC0_SUBTYPE_MASK); } break; case 135: -#line 569 "grammar.y" +#line 583 "grammar.y" { yyval.rblk = gen_p80211_fcdir(cstate, yystack.l_mark[0].i); } break; case 137: -#line 573 "grammar.y" +#line 587 "grammar.y" { yyval.i = str2tok(yystack.l_mark[0].s, ieee80211_types); if (yyval.i == -1) bpf_error(cstate, "unknown 802.11 type name"); } break; case 139: -#line 580 "grammar.y" +#line 594 "grammar.y" { const struct tok *types = NULL; int i; for (i = 0;; i++) { @@ -2034,7 +2048,7 @@ case 139: } break; case 140: -#line 600 "grammar.y" +#line 614 "grammar.y" { int i; for (i = 0;; i++) { if (ieee80211_type_subtypes[i].tok == NULL) { @@ -2051,11 +2065,11 @@ case 140: } break; case 141: -#line 616 "grammar.y" +#line 630 "grammar.y" { yyval.rblk = gen_llc(cstate); } break; case 142: -#line 617 "grammar.y" +#line 631 "grammar.y" { if (pcap_strcasecmp(yystack.l_mark[0].s, "i") == 0) yyval.rblk = gen_llc_i(cstate); else if (pcap_strcasecmp(yystack.l_mark[0].s, "s") == 0) @@ -2078,11 +2092,11 @@ case 142: } break; case 143: -#line 638 "grammar.y" +#line 652 "grammar.y" { yyval.rblk = gen_llc_s_subtype(cstate, LLC_RNR); } break; case 145: -#line 642 "grammar.y" +#line 656 "grammar.y" { if (pcap_strcasecmp(yystack.l_mark[0].s, "nods") == 0) yyval.i = IEEE80211_FC1_DIR_NODS; else if (pcap_strcasecmp(yystack.l_mark[0].s, "tods") == 0) @@ -2096,195 +2110,195 @@ case 145: } break; case 146: -#line 655 "grammar.y" +#line 669 "grammar.y" { yyval.i = yystack.l_mark[0].i; } break; case 147: -#line 656 "grammar.y" +#line 670 "grammar.y" { yyval.i = pfreason_to_num(cstate, yystack.l_mark[0].s); } break; case 148: -#line 659 "grammar.y" +#line 673 "grammar.y" { yyval.i = pfaction_to_num(cstate, yystack.l_mark[0].s); } break; case 149: -#line 662 "grammar.y" +#line 676 "grammar.y" { yyval.i = BPF_JGT; } break; case 150: -#line 663 "grammar.y" +#line 677 "grammar.y" { yyval.i = BPF_JGE; } break; case 151: -#line 664 "grammar.y" +#line 678 "grammar.y" { yyval.i = BPF_JEQ; } break; case 152: -#line 666 "grammar.y" +#line 680 "grammar.y" { yyval.i = BPF_JGT; } break; case 153: -#line 667 "grammar.y" +#line 681 "grammar.y" { yyval.i = BPF_JGE; } break; case 154: -#line 668 "grammar.y" +#line 682 "grammar.y" { yyval.i = BPF_JEQ; } break; case 155: -#line 670 "grammar.y" +#line 684 "grammar.y" { yyval.a = gen_loadi(cstate, yystack.l_mark[0].i); } break; case 157: -#line 673 "grammar.y" +#line 687 "grammar.y" { yyval.a = gen_load(cstate, yystack.l_mark[-3].i, yystack.l_mark[-1].a, 1); } break; case 158: -#line 674 "grammar.y" +#line 688 "grammar.y" { yyval.a = gen_load(cstate, yystack.l_mark[-5].i, yystack.l_mark[-3].a, yystack.l_mark[-1].i); } break; case 159: -#line 675 "grammar.y" +#line 689 "grammar.y" { yyval.a = gen_arth(cstate, BPF_ADD, yystack.l_mark[-2].a, yystack.l_mark[0].a); } break; case 160: -#line 676 "grammar.y" +#line 690 "grammar.y" { yyval.a = gen_arth(cstate, BPF_SUB, yystack.l_mark[-2].a, yystack.l_mark[0].a); } break; case 161: -#line 677 "grammar.y" +#line 691 "grammar.y" { yyval.a = gen_arth(cstate, BPF_MUL, yystack.l_mark[-2].a, yystack.l_mark[0].a); } break; case 162: -#line 678 "grammar.y" +#line 692 "grammar.y" { yyval.a = gen_arth(cstate, BPF_DIV, yystack.l_mark[-2].a, yystack.l_mark[0].a); } break; case 163: -#line 679 "grammar.y" +#line 693 "grammar.y" { yyval.a = gen_arth(cstate, BPF_MOD, yystack.l_mark[-2].a, yystack.l_mark[0].a); } break; case 164: -#line 680 "grammar.y" +#line 694 "grammar.y" { yyval.a = gen_arth(cstate, BPF_AND, yystack.l_mark[-2].a, yystack.l_mark[0].a); } break; case 165: -#line 681 "grammar.y" +#line 695 "grammar.y" { yyval.a = gen_arth(cstate, BPF_OR, yystack.l_mark[-2].a, yystack.l_mark[0].a); } break; case 166: -#line 682 "grammar.y" +#line 696 "grammar.y" { yyval.a = gen_arth(cstate, BPF_XOR, yystack.l_mark[-2].a, yystack.l_mark[0].a); } break; case 167: -#line 683 "grammar.y" +#line 697 "grammar.y" { yyval.a = gen_arth(cstate, BPF_LSH, yystack.l_mark[-2].a, yystack.l_mark[0].a); } break; case 168: -#line 684 "grammar.y" +#line 698 "grammar.y" { yyval.a = gen_arth(cstate, BPF_RSH, yystack.l_mark[-2].a, yystack.l_mark[0].a); } break; case 169: -#line 685 "grammar.y" +#line 699 "grammar.y" { yyval.a = gen_neg(cstate, yystack.l_mark[0].a); } break; case 170: -#line 686 "grammar.y" +#line 700 "grammar.y" { yyval.a = yystack.l_mark[-1].a; } break; case 171: -#line 687 "grammar.y" +#line 701 "grammar.y" { yyval.a = gen_loadlen(cstate); } break; case 172: -#line 689 "grammar.y" +#line 703 "grammar.y" { yyval.i = '&'; } break; case 173: -#line 690 "grammar.y" +#line 704 "grammar.y" { yyval.i = '|'; } break; case 174: -#line 691 "grammar.y" +#line 705 "grammar.y" { yyval.i = '<'; } break; case 175: -#line 692 "grammar.y" +#line 706 "grammar.y" { yyval.i = '>'; } break; case 176: -#line 693 "grammar.y" +#line 707 "grammar.y" { yyval.i = '='; } break; case 178: -#line 696 "grammar.y" +#line 710 "grammar.y" { yyval.i = yystack.l_mark[-1].i; } break; case 179: -#line 698 "grammar.y" +#line 712 "grammar.y" { yyval.i = A_LANE; } break; case 180: -#line 699 "grammar.y" +#line 713 "grammar.y" { yyval.i = A_METAC; } break; case 181: -#line 700 "grammar.y" +#line 714 "grammar.y" { yyval.i = A_BCC; } break; case 182: -#line 701 "grammar.y" +#line 715 "grammar.y" { yyval.i = A_OAMF4EC; } break; case 183: -#line 702 "grammar.y" +#line 716 "grammar.y" { yyval.i = A_OAMF4SC; } break; case 184: -#line 703 "grammar.y" +#line 717 "grammar.y" { yyval.i = A_SC; } break; case 185: -#line 704 "grammar.y" +#line 718 "grammar.y" { yyval.i = A_ILMIC; } break; case 186: -#line 706 "grammar.y" +#line 720 "grammar.y" { yyval.i = A_OAM; } break; case 187: -#line 707 "grammar.y" +#line 721 "grammar.y" { yyval.i = A_OAMF4; } break; case 188: -#line 708 "grammar.y" +#line 722 "grammar.y" { yyval.i = A_CONNECTMSG; } break; case 189: -#line 709 "grammar.y" +#line 723 "grammar.y" { yyval.i = A_METACONNECT; } break; case 190: -#line 712 "grammar.y" +#line 726 "grammar.y" { yyval.blk.atmfieldtype = A_VPI; } break; case 191: -#line 713 "grammar.y" +#line 727 "grammar.y" { yyval.blk.atmfieldtype = A_VCI; } break; case 193: -#line 716 "grammar.y" +#line 730 "grammar.y" { yyval.blk.b = gen_atmfield_code(cstate, yystack.l_mark[-2].blk.atmfieldtype, (bpf_int32)yystack.l_mark[0].i, (bpf_u_int32)yystack.l_mark[-1].i, 0); } break; case 194: -#line 717 "grammar.y" +#line 731 "grammar.y" { yyval.blk.b = gen_atmfield_code(cstate, yystack.l_mark[-2].blk.atmfieldtype, (bpf_int32)yystack.l_mark[0].i, (bpf_u_int32)yystack.l_mark[-1].i, 1); } break; case 195: -#line 718 "grammar.y" +#line 732 "grammar.y" { yyval.blk.b = yystack.l_mark[-1].blk.b; yyval.blk.q = qerr; } break; case 196: -#line 720 "grammar.y" +#line 734 "grammar.y" { yyval.blk.atmfieldtype = yystack.l_mark[-1].blk.atmfieldtype; if (yyval.blk.atmfieldtype == A_VPI || @@ -2293,79 +2307,79 @@ case 196: } break; case 198: -#line 728 "grammar.y" +#line 742 "grammar.y" { gen_or(yystack.l_mark[-2].blk.b, yystack.l_mark[0].blk.b); yyval.blk = yystack.l_mark[0].blk; } break; case 199: -#line 731 "grammar.y" +#line 745 "grammar.y" { yyval.i = M_FISU; } break; case 200: -#line 732 "grammar.y" +#line 746 "grammar.y" { yyval.i = M_LSSU; } break; case 201: -#line 733 "grammar.y" +#line 747 "grammar.y" { yyval.i = M_MSU; } break; case 202: -#line 734 "grammar.y" +#line 748 "grammar.y" { yyval.i = MH_FISU; } break; case 203: -#line 735 "grammar.y" +#line 749 "grammar.y" { yyval.i = MH_LSSU; } break; case 204: -#line 736 "grammar.y" +#line 750 "grammar.y" { yyval.i = MH_MSU; } break; case 205: -#line 739 "grammar.y" +#line 753 "grammar.y" { yyval.blk.mtp3fieldtype = M_SIO; } break; case 206: -#line 740 "grammar.y" +#line 754 "grammar.y" { yyval.blk.mtp3fieldtype = M_OPC; } break; case 207: -#line 741 "grammar.y" +#line 755 "grammar.y" { yyval.blk.mtp3fieldtype = M_DPC; } break; case 208: -#line 742 "grammar.y" +#line 756 "grammar.y" { yyval.blk.mtp3fieldtype = M_SLS; } break; case 209: -#line 743 "grammar.y" +#line 757 "grammar.y" { yyval.blk.mtp3fieldtype = MH_SIO; } break; case 210: -#line 744 "grammar.y" +#line 758 "grammar.y" { yyval.blk.mtp3fieldtype = MH_OPC; } break; case 211: -#line 745 "grammar.y" +#line 759 "grammar.y" { yyval.blk.mtp3fieldtype = MH_DPC; } break; case 212: -#line 746 "grammar.y" +#line 760 "grammar.y" { yyval.blk.mtp3fieldtype = MH_SLS; } break; case 214: -#line 749 "grammar.y" +#line 763 "grammar.y" { yyval.blk.b = gen_mtp3field_code(cstate, yystack.l_mark[-2].blk.mtp3fieldtype, (u_int)yystack.l_mark[0].i, (u_int)yystack.l_mark[-1].i, 0); } break; case 215: -#line 750 "grammar.y" +#line 764 "grammar.y" { yyval.blk.b = gen_mtp3field_code(cstate, yystack.l_mark[-2].blk.mtp3fieldtype, (u_int)yystack.l_mark[0].i, (u_int)yystack.l_mark[-1].i, 1); } break; case 216: -#line 751 "grammar.y" +#line 765 "grammar.y" { yyval.blk.b = yystack.l_mark[-1].blk.b; yyval.blk.q = qerr; } break; case 217: -#line 753 "grammar.y" +#line 767 "grammar.y" { yyval.blk.mtp3fieldtype = yystack.l_mark[-1].blk.mtp3fieldtype; if (yyval.blk.mtp3fieldtype == M_SIO || @@ -2380,10 +2394,10 @@ case 217: } break; case 219: -#line 767 "grammar.y" +#line 781 "grammar.y" { gen_or(yystack.l_mark[-2].blk.b, yystack.l_mark[0].blk.b); yyval.blk = yystack.l_mark[0].blk; } break; -#line 2387 "grammar.c" +#line 2401 "grammar.c" } yystack.s_mark -= yym; yystate = *yystack.s_mark; diff --git a/freebsd/contrib/libpcap/grammar.y b/freebsd/contrib/libpcap/grammar.y index ef6a7abb..be80e2bf 100644 --- a/freebsd/contrib/libpcap/grammar.y +++ b/freebsd/contrib/libpcap/grammar.y @@ -21,7 +21,7 @@ /* * And we need to pass the compiler state to the scanner. */ -%parse-param {compiler_state_t *cstate} +%parse-param { compiler_state_t *cstate } %{ /* @@ -44,23 +44,18 @@ * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. * - * $FreeBSD$ */ #ifdef HAVE_CONFIG_H -#include "config.h" +#include <config.h> #endif -#ifdef _WIN32 -#include <pcap-stdinc.h> -#else /* _WIN32 */ -#include <sys/types.h> -#include <sys/socket.h> -#endif /* _WIN32 */ - #include <stdlib.h> #ifndef _WIN32 +#include <sys/types.h> +#include <sys/socket.h> + #if __STDC__ struct mbuf; struct rtentry; @@ -72,6 +67,8 @@ struct rtentry; #include <stdio.h> +#include "diag-control.h" + #include "pcap-int.h" #include "gencode.h" @@ -80,7 +77,7 @@ struct rtentry; #ifdef HAVE_NET_PFVAR_H #include <net/if.h> -#include <netpfil/pf/pf.h> +#include <net/pfvar.h> #include <net/if_pflog.h> #endif #include "llc.h" @@ -91,9 +88,30 @@ struct rtentry; #include "os-proto.h" #endif -#define QSET(q, p, d, a) (q).proto = (p),\ - (q).dir = (d),\ - (q).addr = (a) +#ifdef YYBYACC +/* + * Both Berkeley YACC and Bison define yydebug (under whatever name + * it has) as a global, but Bison does so only if YYDEBUG is defined. + * Berkeley YACC define it even if YYDEBUG isn't defined; declare it + * here to suppress a warning. + */ +#if !defined(YYDEBUG) +extern int yydebug; +#endif + +/* + * In Berkeley YACC, yynerrs (under whatever name it has) is global, + * even if it's building a reentrant parser. In Bison, it's local + * in reentrant parsers. + * + * Declare it to squelch a warning. + */ +extern int yynerrs; +#endif + +#define QSET(q, p, d, a) (q).proto = (unsigned char)(p),\ + (q).dir = (unsigned char)(d),\ + (q).addr = (unsigned char)(a) struct tok { int v; /* value */ @@ -200,8 +218,8 @@ str2tok(const char *str, const struct tok *toks) static struct qual qerr = { Q_UNDEF, Q_UNDEF, Q_UNDEF, Q_UNDEF }; -static void -yyerror(void *yyscanner, compiler_state_t *cstate, const char *msg) +static PCAP_NORETURN_DEF void +yyerror(void *yyscanner _U_, compiler_state_t *cstate, const char *msg) { bpf_syntax_error(cstate, msg); /* NOTREACHED */ @@ -247,26 +265,22 @@ pfaction_to_num(compiler_state_t *cstate, const char *action) } } #else /* !HAVE_NET_PFVAR_H */ -static int -pfreason_to_num(compiler_state_t *cstate, const char *reason) +static PCAP_NORETURN_DEF int +pfreason_to_num(compiler_state_t *cstate, const char *reason _U_) { bpf_error(cstate, "libpcap was compiled on a machine without pf support"); /*NOTREACHED*/ - - /* this is to make the VC compiler happy */ - return -1; } -static int -pfaction_to_num(compiler_state_t *cstate, const char *action) +static PCAP_NORETURN_DEF int +pfaction_to_num(compiler_state_t *cstate, const char *action _U_) { bpf_error(cstate, "libpcap was compiled on a machine without pf support"); /*NOTREACHED*/ - - /* this is to make the VC compiler happy */ - return -1; } #endif /* HAVE_NET_PFVAR_H */ + +DIAG_OFF_BISON_BYACC %} %union { diff --git a/freebsd/contrib/libpcap/ieee80211.h b/freebsd/contrib/libpcap/ieee80211.h index 894a9e76..d79f0f8e 100644 --- a/freebsd/contrib/libpcap/ieee80211.h +++ b/freebsd/contrib/libpcap/ieee80211.h @@ -90,7 +90,7 @@ #define IEEE80211_FC1_RETRY 0x08 #define IEEE80211_FC1_PWR_MGT 0x10 #define IEEE80211_FC1_MORE_DATA 0x20 -#define IEEE80211_FC1_PROTECTED 0x40 +#define IEEE80211_FC1_WEP 0x40 #define IEEE80211_FC1_ORDER 0x80 #define IEEE80211_SEQ_FRAG_MASK 0x000f diff --git a/freebsd/contrib/libpcap/inet.c b/freebsd/contrib/libpcap/inet.c deleted file mode 100644 index 1945e4fc..00000000 --- a/freebsd/contrib/libpcap/inet.c +++ /dev/null @@ -1,409 +0,0 @@ -#include <machine/rtems-bsd-user-space.h> - -/* -*- Mode: c; tab-width: 8; indent-tabs-mode: 1; c-basic-offset: 8; -*- */ -/* - * Copyright (c) 1994, 1995, 1996, 1997, 1998 - * The Regents of the University of California. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the Computer Systems - * Engineering Group at Lawrence Berkeley Laboratory. - * 4. Neither the name of the University nor of the Laboratory may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#ifdef _WIN32 -#include <pcap-stdinc.h> -#else /* _WIN32 */ - -#include <sys/param.h> -#ifndef MSDOS -#include <sys/file.h> -#endif -#include <sys/ioctl.h> -#include <sys/socket.h> -#ifdef HAVE_SYS_SOCKIO_H -#include <sys/sockio.h> -#endif - -struct mbuf; /* Squelch compiler warnings on some platforms for */ -struct rtentry; /* declarations in <net/if.h> */ -#include <net/if.h> -#include <netinet/in.h> -#endif /* _WIN32 */ - -#include <errno.h> -#include <memory.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#if !defined(_WIN32) && !defined(__BORLANDC__) -#include <unistd.h> -#endif /* !_WIN32 && !__BORLANDC__ */ - -#include "pcap-int.h" - -#ifdef HAVE_OS_PROTO_H -#include "os-proto.h" -#endif - -#if !defined(_WIN32) && !defined(MSDOS) - -/* - * Return the name of a network interface attached to the system, or NULL - * if none can be found. The interface must be configured up; the - * lowest unit number is preferred; loopback is ignored. - */ -char * -pcap_lookupdev(errbuf) - register char *errbuf; -{ - pcap_if_t *alldevs; -/* for old BSD systems, including bsdi3 */ -#ifndef IF_NAMESIZE -#define IF_NAMESIZE IFNAMSIZ -#endif - static char device[IF_NAMESIZE + 1]; - char *ret; - - if (pcap_findalldevs(&alldevs, errbuf) == -1) - return (NULL); - - if (alldevs == NULL || (alldevs->flags & PCAP_IF_LOOPBACK)) { - /* - * There are no devices on the list, or the first device - * on the list is a loopback device, which means there - * are no non-loopback devices on the list. This means - * we can't return any device. - * - * XXX - why not return a loopback device? If we can't - * capture on it, it won't be on the list, and if it's - * on the list, there aren't any non-loopback devices, - * so why not just supply it as the default device? - */ - (void)strlcpy(errbuf, "no suitable device found", - PCAP_ERRBUF_SIZE); - ret = NULL; - } else { - /* - * Return the name of the first device on the list. - */ - (void)strlcpy(device, alldevs->name, sizeof(device)); - ret = device; - } - - pcap_freealldevs(alldevs); - return (ret); -} - -int -pcap_lookupnet(device, netp, maskp, errbuf) - register const char *device; - register bpf_u_int32 *netp, *maskp; - register char *errbuf; -{ - register int fd; - register struct sockaddr_in *sin4; - struct ifreq ifr; - - /* - * The pseudo-device "any" listens on all interfaces and therefore - * has the network address and -mask "0.0.0.0" therefore catching - * all traffic. Using NULL for the interface is the same as "any". - */ - if (!device || strcmp(device, "any") == 0 -#ifdef HAVE_DAG_API - || strstr(device, "dag") != NULL -#endif -#ifdef HAVE_SEPTEL_API - || strstr(device, "septel") != NULL -#endif -#ifdef PCAP_SUPPORT_BT - || strstr(device, "bluetooth") != NULL -#endif -#ifdef PCAP_SUPPORT_USB - || strstr(device, "usbmon") != NULL -#endif -#ifdef PCAP_SUPPORT_NETMAP - || !strncmp(device, "netmap:", 7) - || !strncmp(device, "vale", 4) -#endif -#ifdef HAVE_SNF_API - || strstr(device, "snf") != NULL -#endif - ) { - *netp = *maskp = 0; - return 0; - } - - fd = socket(AF_INET, SOCK_DGRAM, 0); - if (fd < 0) { - (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, "socket: %s", - pcap_strerror(errno)); - return (-1); - } - memset(&ifr, 0, sizeof(ifr)); -#ifdef linux - /* XXX Work around Linux kernel bug */ - ifr.ifr_addr.sa_family = AF_INET; -#endif - (void)strlcpy(ifr.ifr_name, device, sizeof(ifr.ifr_name)); - if (ioctl(fd, SIOCGIFADDR, (char *)&ifr) < 0) { - if (errno == EADDRNOTAVAIL) { - (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "%s: no IPv4 address assigned", device); - } else { - (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "SIOCGIFADDR: %s: %s", - device, pcap_strerror(errno)); - } - (void)close(fd); - return (-1); - } - sin4 = (struct sockaddr_in *)&ifr.ifr_addr; - *netp = sin4->sin_addr.s_addr; - memset(&ifr, 0, sizeof(ifr)); -#ifdef linux - /* XXX Work around Linux kernel bug */ - ifr.ifr_addr.sa_family = AF_INET; -#endif - (void)strlcpy(ifr.ifr_name, device, sizeof(ifr.ifr_name)); - if (ioctl(fd, SIOCGIFNETMASK, (char *)&ifr) < 0) { - (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "SIOCGIFNETMASK: %s: %s", device, pcap_strerror(errno)); - (void)close(fd); - return (-1); - } - (void)close(fd); - *maskp = sin4->sin_addr.s_addr; - if (*maskp == 0) { - if (IN_CLASSA(*netp)) - *maskp = IN_CLASSA_NET; - else if (IN_CLASSB(*netp)) - *maskp = IN_CLASSB_NET; - else if (IN_CLASSC(*netp)) - *maskp = IN_CLASSC_NET; - else { - (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "inet class for 0x%x unknown", *netp); - return (-1); - } - } - *netp &= *maskp; - return (0); -} - -#elif defined(_WIN32) - -/* - * Return the name of a network interface attached to the system, or NULL - * if none can be found. The interface must be configured up; the - * lowest unit number is preferred; loopback is ignored. - * - * In the best of all possible worlds, this would be the same as on - * UN*X, but there may be software that expects this to return a - * full list of devices after the first device. - */ -#define ADAPTERSNAME_LEN 8192 -char * -pcap_lookupdev(errbuf) - register char *errbuf; -{ - DWORD dwVersion; - DWORD dwWindowsMajorVersion; - char our_errbuf[PCAP_ERRBUF_SIZE+1]; - -#pragma warning (push) -#pragma warning (disable: 4996) /* disable MSVC's GetVersion() deprecated warning here */ - dwVersion = GetVersion(); /* get the OS version */ -#pragma warning (pop) - dwWindowsMajorVersion = (DWORD)(LOBYTE(LOWORD(dwVersion))); - - if (dwVersion >= 0x80000000 && dwWindowsMajorVersion >= 4) { - /* - * Windows 95, 98, ME. - */ - ULONG NameLength = ADAPTERSNAME_LEN; - static char AdaptersName[ADAPTERSNAME_LEN]; - - if (PacketGetAdapterNames(AdaptersName,&NameLength) ) - return (AdaptersName); - else - return NULL; - } else { - /* - * Windows NT (NT 4.0 and later). - * Convert the names to Unicode for backward compatibility. - */ - ULONG NameLength = ADAPTERSNAME_LEN; - static WCHAR AdaptersName[ADAPTERSNAME_LEN]; - size_t BufferSpaceLeft; - char *tAstr; - WCHAR *Unameptr; - char *Adescptr; - size_t namelen, i; - WCHAR *TAdaptersName = (WCHAR*)malloc(ADAPTERSNAME_LEN * sizeof(WCHAR)); - int NAdapts = 0; - - if(TAdaptersName == NULL) - { - (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, "memory allocation failure"); - return NULL; - } - - if ( !PacketGetAdapterNames((PTSTR)TAdaptersName,&NameLength) ) - { - pcap_win32_err_to_str(GetLastError(), our_errbuf); - (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "PacketGetAdapterNames: %s", our_errbuf); - free(TAdaptersName); - return NULL; - } - - - BufferSpaceLeft = ADAPTERSNAME_LEN * sizeof(WCHAR); - tAstr = (char*)TAdaptersName; - Unameptr = AdaptersName; - - /* - * Convert the device names to Unicode into AdapterName. - */ - do { - /* - * Length of the name, including the terminating - * NUL. - */ - namelen = strlen(tAstr) + 1; - - /* - * Do we have room for the name in the Unicode - * buffer? - */ - if (BufferSpaceLeft < namelen * sizeof(WCHAR)) { - /* - * No. - */ - goto quit; - } - BufferSpaceLeft -= namelen * sizeof(WCHAR); - - /* - * Copy the name, converting ASCII to Unicode. - * namelen includes the NUL, so we copy it as - * well. - */ - for (i = 0; i < namelen; i++) - *Unameptr++ = *tAstr++; - - /* - * Count this adapter. - */ - NAdapts++; - } while (namelen != 1); - - /* - * Copy the descriptions, but don't convert them from - * ASCII to Unicode. - */ - Adescptr = (char *)Unameptr; - while(NAdapts--) - { - size_t desclen; - - desclen = strlen(tAstr) + 1; - - /* - * Do we have room for the name in the Unicode - * buffer? - */ - if (BufferSpaceLeft < desclen) { - /* - * No. - */ - goto quit; - } - - /* - * Just copy the ASCII string. - * namelen includes the NUL, so we copy it as - * well. - */ - memcpy(Adescptr, tAstr, desclen); - Adescptr += desclen; - tAstr += desclen; - BufferSpaceLeft -= desclen; - } - - quit: - free(TAdaptersName); - return (char *)(AdaptersName); - } -} - - -int -pcap_lookupnet(device, netp, maskp, errbuf) - register const char *device; - register bpf_u_int32 *netp, *maskp; - register char *errbuf; -{ - /* - * We need only the first IPv4 address, so we must scan the array returned by PacketGetNetInfo() - * in order to skip non IPv4 (i.e. IPv6 addresses) - */ - npf_if_addr if_addrs[MAX_NETWORK_ADDRESSES]; - LONG if_addr_size = 1; - struct sockaddr_in *t_addr; - unsigned int i; - - if (!PacketGetNetInfoEx((void *)device, if_addrs, &if_addr_size)) { - *netp = *maskp = 0; - return (0); - } - - for(i=0; i<MAX_NETWORK_ADDRESSES; i++) - { - if(if_addrs[i].IPAddress.ss_family == AF_INET) - { - t_addr = (struct sockaddr_in *) &(if_addrs[i].IPAddress); - *netp = t_addr->sin_addr.S_un.S_addr; - t_addr = (struct sockaddr_in *) &(if_addrs[i].SubnetMask); - *maskp = t_addr->sin_addr.S_un.S_addr; - - *netp &= *maskp; - return (0); - } - - } - - *netp = *maskp = 0; - return (0); -} - -#endif /* !_WIN32 && !MSDOS */ diff --git a/freebsd/contrib/libpcap/nametoaddr.c b/freebsd/contrib/libpcap/nametoaddr.c index 7ef8f4cc..1d47ff9d 100644 --- a/freebsd/contrib/libpcap/nametoaddr.c +++ b/freebsd/contrib/libpcap/nametoaddr.c @@ -22,12 +22,10 @@ * * Name to id translation routines used by the scanner. * These functions are not time critical. - * - * $FreeBSD$ */ #ifdef HAVE_CONFIG_H -#include "config.h" +#include <config.h> #endif #ifdef DECNETLIB @@ -36,62 +34,99 @@ #endif #ifdef _WIN32 -#include <pcap-stdinc.h> - -#ifdef INET6 -/* - * To quote the MSDN page for getaddrinfo() at - * - * https://msdn.microsoft.com/en-us/library/windows/desktop/ms738520(v=vs.85).aspx - * - * "Support for getaddrinfo on Windows 2000 and older versions - * The getaddrinfo function was added to the Ws2_32.dll on Windows XP and - * later. To execute an application that uses this function on earlier - * versions of Windows, then you need to include the Ws2tcpip.h and - * Wspiapi.h files. When the Wspiapi.h include file is added, the - * getaddrinfo function is defined to the WspiapiGetAddrInfo inline - * function in the Wspiapi.h file. At runtime, the WspiapiGetAddrInfo - * function is implemented in such a way that if the Ws2_32.dll or the - * Wship6.dll (the file containing getaddrinfo in the IPv6 Technology - * Preview for Windows 2000) does not include getaddrinfo, then a - * version of getaddrinfo is implemented inline based on code in the - * Wspiapi.h header file. This inline code will be used on older Windows - * platforms that do not natively support the getaddrinfo function." - * - * We use getaddrinfo(), so we include Wspiapi.h here. pcap-stdinc.h - * includes Ws2tcpip.h, so we don't need to include it ourselves. - */ -#include <Wspiapi.h> -#endif - + #include <winsock2.h> + #include <ws2tcpip.h> + + #ifdef INET6 + /* + * To quote the MSDN page for getaddrinfo() at + * + * https://msdn.microsoft.com/en-us/library/windows/desktop/ms738520(v=vs.85).aspx + * + * "Support for getaddrinfo on Windows 2000 and older versions + * The getaddrinfo function was added to the Ws2_32.dll on Windows XP and + * later. To execute an application that uses this function on earlier + * versions of Windows, then you need to include the Ws2tcpip.h and + * Wspiapi.h files. When the Wspiapi.h include file is added, the + * getaddrinfo function is defined to the WspiapiGetAddrInfo inline + * function in the Wspiapi.h file. At runtime, the WspiapiGetAddrInfo + * function is implemented in such a way that if the Ws2_32.dll or the + * Wship6.dll (the file containing getaddrinfo in the IPv6 Technology + * Preview for Windows 2000) does not include getaddrinfo, then a + * version of getaddrinfo is implemented inline based on code in the + * Wspiapi.h header file. This inline code will be used on older Windows + * platforms that do not natively support the getaddrinfo function." + * + * We use getaddrinfo(), so we include Wspiapi.h here. + */ + #include <wspiapi.h> + #endif /* INET6 */ #else /* _WIN32 */ - -#include <sys/param.h> -#include <sys/types.h> /* concession to AIX */ -#include <sys/socket.h> -#include <sys/time.h> - -#include <netinet/in.h> -#endif /* _WIN32 */ - -#ifndef _WIN32 -#ifdef HAVE_ETHER_HOSTTON -/* - * XXX - do we need any of this if <netinet/if_ether.h> doesn't declare - * ether_hostton()? - */ -#ifdef HAVE_NETINET_IF_ETHER_H -struct mbuf; /* Squelch compiler warnings on some platforms for */ -struct rtentry; /* declarations in <net/if.h> */ -#include <net/if.h> /* for "struct ifnet" in "struct arpcom" on Solaris */ -#include <netinet/if_ether.h> -#endif /* HAVE_NETINET_IF_ETHER_H */ -#ifdef NETINET_ETHER_H_DECLARES_ETHER_HOSTTON -#include <netinet/ether.h> -#endif /* NETINET_ETHER_H_DECLARES_ETHER_HOSTTON */ -#endif /* HAVE_ETHER_HOSTTON */ -#include <arpa/inet.h> -#include <netdb.h> + #include <sys/param.h> + #include <sys/types.h> + #include <sys/socket.h> + #include <sys/time.h> + + #include <netinet/in.h> + + #ifdef HAVE_ETHER_HOSTTON + #if defined(NET_ETHERNET_H_DECLARES_ETHER_HOSTTON) + /* + * OK, just include <net/ethernet.h>. + */ + #include <net/ethernet.h> + #elif defined(NETINET_ETHER_H_DECLARES_ETHER_HOSTTON) + /* + * OK, just include <netinet/ether.h> + */ + #include <netinet/ether.h> + #elif defined(SYS_ETHERNET_H_DECLARES_ETHER_HOSTTON) + /* + * OK, just include <sys/ethernet.h> + */ + #include <sys/ethernet.h> + #elif defined(ARPA_INET_H_DECLARES_ETHER_HOSTTON) + /* + * OK, just include <arpa/inet.h> + */ + #include <arpa/inet.h> + #elif defined(NETINET_IF_ETHER_H_DECLARES_ETHER_HOSTTON) + /* + * OK, include <netinet/if_ether.h>, after all the other stuff we + * need to include or define for its benefit. + */ + #define NEED_NETINET_IF_ETHER_H + #else + /* + * We'll have to declare it ourselves. + * If <netinet/if_ether.h> defines struct ether_addr, include + * it. Otherwise, define it ourselves. + */ + #ifdef HAVE_STRUCT_ETHER_ADDR + #define NEED_NETINET_IF_ETHER_H + #else /* HAVE_STRUCT_ETHER_ADDR */ + struct ether_addr { + unsigned char ether_addr_octet[6]; + }; + #endif /* HAVE_STRUCT_ETHER_ADDR */ + #endif /* what declares ether_hostton() */ + + #ifdef NEED_NETINET_IF_ETHER_H + #include <net/if.h> /* Needed on some platforms */ + #include <netinet/in.h> /* Needed on some platforms */ + #include <netinet/if_ether.h> + #endif /* NEED_NETINET_IF_ETHER_H */ + + #ifndef HAVE_DECL_ETHER_HOSTTON + /* + * No header declares it, so declare it ourselves. + */ + extern int ether_hostton(const char *, struct ether_addr *); + #endif /* !defined(HAVE_DECL_ETHER_HOSTTON) */ + #endif /* HAVE_ETHER_HOSTTON */ + + #include <arpa/inet.h> + #include <netdb.h> #endif /* _WIN32 */ #include <ctype.h> @@ -115,11 +150,10 @@ struct rtentry; /* declarations in <net/if.h> */ #define NTOHS(x) (x) = ntohs(x) #endif -static inline int xdtoi(int); - /* * Convert host name to internet address. * Return 0 upon failure. + * XXX - not thread-safe; don't use it inside libpcap. */ bpf_u_int32 ** pcap_nametoaddr(const char *name) @@ -145,7 +179,6 @@ pcap_nametoaddr(const char *name) return 0; } -#ifdef INET6 struct addrinfo * pcap_nametoaddrinfo(const char *name) { @@ -162,23 +195,17 @@ pcap_nametoaddrinfo(const char *name) else return res; } -#endif /*INET6*/ /* * Convert net name to internet address. * Return 0 upon failure. + * XXX - not guaranteed to be thread-safe! See below for platforms + * on which it is thread-safe and on which it isn't. */ bpf_u_int32 pcap_nametonetaddr(const char *name) { -#ifndef _WIN32 - struct netent *np; - - if ((np = getnetbyname(name)) != NULL) - return np->n_net; - else - return 0; -#else +#ifdef _WIN32 /* * There's no "getnetbyname()" on Windows. * @@ -192,7 +219,66 @@ pcap_nametonetaddr(const char *name) * of *UN*X* machines.) */ return 0; -#endif +#else + /* + * UN*X. + */ + struct netent *np; + #if defined(HAVE_LINUX_GETNETBYNAME_R) + /* + * We have Linux's reentrant getnetbyname_r(). + */ + struct netent result_buf; + char buf[1024]; /* arbitrary size */ + int h_errnoval; + int err; + + err = getnetbyname_r(name, &result_buf, buf, sizeof buf, &np, + &h_errnoval); + if (err != 0) { + /* + * XXX - dynamically allocate the buffer, and make it + * bigger if we get ERANGE back? + */ + return 0; + } + #elif defined(HAVE_SOLARIS_IRIX_GETNETBYNAME_R) + /* + * We have Solaris's and IRIX's reentrant getnetbyname_r(). + */ + struct netent result_buf; + char buf[1024]; /* arbitrary size */ + + np = getnetbyname_r(name, &result_buf, buf, (int)sizeof buf); + #elif defined(HAVE_AIX_GETNETBYNAME_R) + /* + * We have AIX's reentrant getnetbyname_r(). + */ + struct netent result_buf; + struct netent_data net_data; + + if (getnetbyname_r(name, &result_buf, &net_data) == -1) + np = NULL; + else + np = &result_buf; + #else + /* + * We don't have any getnetbyname_r(); either we have a + * getnetbyname() that uses thread-specific data, in which + * case we're thread-safe (sufficiently recent FreeBSD, + * sufficiently recent Darwin-based OS, sufficiently recent + * HP-UX, sufficiently recent Tru64 UNIX), or we have the + * traditional getnetbyname() (everything else, including + * current NetBSD and OpenBSD), in which case we're not + * thread-safe. + */ + np = getnetbyname(name); + #endif + if (np != NULL) + return np->n_net; + else + return 0; +#endif /* _WIN32 */ } /* @@ -203,20 +289,111 @@ pcap_nametonetaddr(const char *name) int pcap_nametoport(const char *name, int *port, int *proto) { - struct servent *sp; + struct addrinfo hints, *res, *ai; + int error; + struct sockaddr_in *in4; +#ifdef INET6 + struct sockaddr_in6 *in6; +#endif int tcp_port = -1; int udp_port = -1; /* + * We check for both TCP and UDP in case there are + * ambiguous entries. + */ + memset(&hints, 0, sizeof(hints)); + hints.ai_family = PF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_protocol = IPPROTO_TCP; + error = getaddrinfo(NULL, name, &hints, &res); + if (error != 0) { + if (error != EAI_NONAME) { + /* + * This is a real error, not just "there's + * no such service name". + * XXX - this doesn't return an error string. + */ + return 0; + } + } else { + /* + * OK, we found it. Did it find anything? + */ + for (ai = res; ai != NULL; ai = ai->ai_next) { + /* + * Does it have an address? + */ + if (ai->ai_addr != NULL) { + /* + * Yes. Get a port number; we're done. + */ + if (ai->ai_addr->sa_family == AF_INET) { + in4 = (struct sockaddr_in *)ai->ai_addr; + tcp_port = ntohs(in4->sin_port); + break; + } +#ifdef INET6 + if (ai->ai_addr->sa_family == AF_INET6) { + in6 = (struct sockaddr_in6 *)ai->ai_addr; + tcp_port = ntohs(in6->sin6_port); + break; + } +#endif + } + } + freeaddrinfo(res); + } + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = PF_UNSPEC; + hints.ai_socktype = SOCK_DGRAM; + hints.ai_protocol = IPPROTO_UDP; + error = getaddrinfo(NULL, name, &hints, &res); + if (error != 0) { + if (error != EAI_NONAME) { + /* + * This is a real error, not just "there's + * no such service name". + * XXX - this doesn't return an error string. + */ + return 0; + } + } else { + /* + * OK, we found it. Did it find anything? + */ + for (ai = res; ai != NULL; ai = ai->ai_next) { + /* + * Does it have an address? + */ + if (ai->ai_addr != NULL) { + /* + * Yes. Get a port number; we're done. + */ + if (ai->ai_addr->sa_family == AF_INET) { + in4 = (struct sockaddr_in *)ai->ai_addr; + udp_port = ntohs(in4->sin_port); + break; + } +#ifdef INET6 + if (ai->ai_addr->sa_family == AF_INET6) { + in6 = (struct sockaddr_in6 *)ai->ai_addr; + udp_port = ntohs(in6->sin6_port); + break; + } +#endif + } + } + freeaddrinfo(res); + } + + /* * We need to check /etc/services for ambiguous entries. - * If we find the ambiguous entry, and it has the + * If we find an ambiguous entry, and it has the * same port number, change the proto to PROTO_UNDEF * so both TCP and UDP will be checked. */ - sp = getservbyname(name, "tcp"); - if (sp != NULL) tcp_port = ntohs(sp->s_port); - sp = getservbyname(name, "udp"); - if (sp != NULL) udp_port = ntohs(sp->s_port); if (tcp_port >= 0) { *port = tcp_port; *proto = IPPROTO_TCP; @@ -295,12 +472,62 @@ pcap_nametoportrange(const char *name, int *port1, int *port2, int *proto) return 1; } +/* + * XXX - not guaranteed to be thread-safe! See below for platforms + * on which it is thread-safe and on which it isn't. + */ int pcap_nametoproto(const char *str) { struct protoent *p; + #if defined(HAVE_LINUX_GETNETBYNAME_R) + /* + * We have Linux's reentrant getprotobyname_r(). + */ + struct protoent result_buf; + char buf[1024]; /* arbitrary size */ + int err; + + err = getprotobyname_r(str, &result_buf, buf, sizeof buf, &p); + if (err != 0) { + /* + * XXX - dynamically allocate the buffer, and make it + * bigger if we get ERANGE back? + */ + return 0; + } + #elif defined(HAVE_SOLARIS_IRIX_GETNETBYNAME_R) + /* + * We have Solaris's and IRIX's reentrant getprotobyname_r(). + */ + struct protoent result_buf; + char buf[1024]; /* arbitrary size */ + + p = getprotobyname_r(str, &result_buf, buf, (int)sizeof buf); + #elif defined(HAVE_AIX_GETNETBYNAME_R) + /* + * We have AIX's reentrant getprotobyname_r(). + */ + struct protoent result_buf; + struct protoent_data proto_data; + if (getprotobyname_r(str, &result_buf, &proto_data) == -1) + p = NULL; + else + p = &result_buf; + #else + /* + * We don't have any getprotobyname_r(); either we have a + * getprotobyname() that uses thread-specific data, in which + * case we're thread-safe (sufficiently recent FreeBSD, + * sufficiently recent Darwin-based OS, sufficiently recent + * HP-UX, sufficiently recent Tru64 UNIX, Windows), or we have + * the traditional getprotobyname() (everything else, including + * current NetBSD and OpenBSD), in which case we're not + * thread-safe. + */ p = getprotobyname(str); + #endif if (p != 0) return p->p_proto; else @@ -320,18 +547,14 @@ struct eproto { * Debian, at least, so make it a public symbol, even though we * don't officially export it by declaring it in a header file. * (Programs *should* do this themselves, as tcpdump now does.) + * + * We declare it here, right before defining it, to squelch any + * warnings we might get from compilers about the lack of a + * declaration. */ +PCAP_API struct eproto eproto_db[]; PCAP_API_DEF struct eproto eproto_db[] = { -#if 0 - /* The FreeBSD elf linker generates a request to copy this array - * (including its size) when you link with -lpcap. In order to - * not bump the major version number of this libpcap.so, we need - * to ensure that the array stays the same size. Since PUP is - * likely never seen in real life any more, it's the first to - * be sacrificed (in favor of ip6). - */ { "pup", ETHERTYPE_PUP }, -#endif { "xns", ETHERTYPE_NS }, { "ip", ETHERTYPE_IP }, #ifdef INET6 @@ -393,17 +616,16 @@ pcap_nametollc(const char *s) return PROTO_UNDEF; } -/* Hex digit to integer. */ -static inline int -xdtoi(c) - register int c; +/* Hex digit to 8-bit unsigned integer. */ +static inline u_char +xdtoi(u_char c) { if (isdigit(c)) - return c - '0'; + return (u_char)(c - '0'); else if (islower(c)) - return c - 'a' + 10; + return (u_char)(c - 'a' + 10); else - return c - 'A' + 10; + return (u_char)(c - 'A' + 10); } int @@ -414,7 +636,7 @@ __pcap_atoin(const char *s, bpf_u_int32 *addr) *addr = 0; len = 0; - while (1) { + for (;;) { n = 0; while (*s && *s != '.') n = n * 10 + *s++ - '0'; @@ -462,7 +684,7 @@ u_char * pcap_ether_aton(const char *s) { register u_char *ep, *e; - register u_int d; + register u_char d; e = ep = (u_char *)malloc(6); if (e == NULL) @@ -483,7 +705,11 @@ pcap_ether_aton(const char *s) } #ifndef HAVE_ETHER_HOSTTON -/* Roll our own */ +/* + * Roll our own. + * XXX - not thread-safe, because pcap_next_etherent() isn't thread- + * safe! Needs a mutex or a thread-safe pcap_next_etherent(). + */ u_char * pcap_ether_hostton(const char *name) { @@ -515,17 +741,10 @@ pcap_ether_hostton(const char *name) return (NULL); } #else - -#if !defined(HAVE_DECL_ETHER_HOSTTON) || !HAVE_DECL_ETHER_HOSTTON -#ifndef HAVE_STRUCT_ETHER_ADDR -struct ether_addr { - unsigned char ether_addr_octet[6]; -}; -#endif -extern int ether_hostton(const char *, struct ether_addr *); -#endif - -/* Use the os supplied routines */ +/* + * Use the OS-supplied routine. + * This *should* be thread-safe; the API doesn't have a static buffer. + */ u_char * pcap_ether_hostton(const char *name) { @@ -542,10 +761,13 @@ pcap_ether_hostton(const char *name) } #endif +/* + * XXX - not guaranteed to be thread-safe! + */ int +#ifdef DECNETLIB __pcap_nametodnaddr(const char *name, u_short *res) { -#ifdef DECNETLIB struct nodeent *getnodebyname(); struct nodeent *nep; @@ -556,6 +778,8 @@ __pcap_nametodnaddr(const char *name, u_short *res) memcpy((char *)res, (char *)nep->n_addr, sizeof(unsigned short)); return(1); #else +__pcap_nametodnaddr(const char *name _U_, u_short *res _U_) +{ return(0); #endif } diff --git a/freebsd/contrib/libpcap/optimize.c b/freebsd/contrib/libpcap/optimize.c index b861082d..7d2a1826 100644 --- a/freebsd/contrib/libpcap/optimize.c +++ b/freebsd/contrib/libpcap/optimize.c @@ -20,26 +20,14 @@ * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. * - * Optimization module for tcpdump intermediate representation. + * Optimization module for BPF code intermediate representation. */ #ifdef HAVE_CONFIG_H -#include "config.h" +#include <config.h> #endif -#ifdef _WIN32 -#include <pcap-stdinc.h> -#else /* _WIN32 */ -#if HAVE_INTTYPES_H -#include <inttypes.h> -#elif HAVE_STDINT_H -#include <stdint.h> -#endif -#ifdef HAVE_SYS_BITYPES_H -#include <sys/bitypes.h> -#endif -#include <sys/types.h> -#endif /* _WIN32 */ +#include <pcap-types.h> #include <stdio.h> #include <stdlib.h> @@ -51,39 +39,149 @@ #include "pcap-int.h" #include "gencode.h" +#include "optimize.h" #ifdef HAVE_OS_PROTO_H #include "os-proto.h" #endif #ifdef BDEBUG -int pcap_optimizer_debug; -#endif +/* + * The internal "debug printout" flag for the filter expression optimizer. + * The code to print that stuff is present only if BDEBUG is defined, so + * the flag, and the routine to set it, are defined only if BDEBUG is + * defined. + */ +static int pcap_optimizer_debug; -#if defined(MSDOS) && !defined(__DJGPP__) -extern int _w32_ffs (int mask); -#define ffs _w32_ffs -#endif +/* + * Routine to set that flag. + * + * This is intended for libpcap developers, not for general use. + * If you want to set these in a program, you'll have to declare this + * routine yourself, with the appropriate DLL import attribute on Windows; + * it's not declared in any header file, and won't be declared in any + * header file provided by libpcap. + */ +PCAP_API void pcap_set_optimizer_debug(int value); + +PCAP_API_DEF void +pcap_set_optimizer_debug(int value) +{ + pcap_optimizer_debug = value; +} + +/* + * The internal "print dot graph" flag for the filter expression optimizer. + * The code to print that stuff is present only if BDEBUG is defined, so + * the flag, and the routine to set it, are defined only if BDEBUG is + * defined. + */ +static int pcap_print_dot_graph; /* - * So is the check for _MSC_VER done because MinGW has this? + * Routine to set that flag. + * + * This is intended for libpcap developers, not for general use. + * If you want to set these in a program, you'll have to declare this + * routine yourself, with the appropriate DLL import attribute on Windows; + * it's not declared in any header file, and won't be declared in any + * header file provided by libpcap. */ -#if defined(_WIN32) && defined (_MSC_VER) +PCAP_API void pcap_set_print_dot_graph(int value); + +PCAP_API_DEF void +pcap_set_print_dot_graph(int value) +{ + pcap_print_dot_graph = value; +} + +#endif + /* - * ffs -- vax ffs instruction + * lowest_set_bit(). + * + * Takes a 32-bit integer as an argument. * - * XXX - with versions of VS that have it, use _BitScanForward()? + * If handed a non-zero value, returns the index of the lowest set bit, + * counting upwards fro zero. + * + * If handed zero, the results are platform- and compiler-dependent. + * Keep it out of the light, don't give it any water, don't feed it + * after midnight, and don't pass zero to it. + * + * This is the same as the count of trailing zeroes in the word. + */ +#if PCAP_IS_AT_LEAST_GNUC_VERSION(3,4) + /* + * GCC 3.4 and later; we have __builtin_ctz(). + */ + #define lowest_set_bit(mask) __builtin_ctz(mask) +#elif defined(_MSC_VER) + /* + * Visual Studio; we support only 2005 and later, so use + * _BitScanForward(). + */ +#include <intrin.h> + +#ifndef __clang__ +#pragma intrinsic(_BitScanForward) +#endif + +static __forceinline int +lowest_set_bit(int mask) +{ + unsigned long bit; + + /* + * Don't sign-extend mask if long is longer than int. + * (It's currently not, in MSVC, even on 64-bit platforms, but....) + */ + if (_BitScanForward(&bit, (unsigned int)mask) == 0) + return -1; /* mask is zero */ + return (int)bit; +} +#elif defined(MSDOS) && defined(__DJGPP__) + /* + * MS-DOS with DJGPP, which declares ffs() in <string.h>, which + * we've already included. + */ + #define lowest_set_bit(mask) (ffs((mask)) - 1) +#elif (defined(MSDOS) && defined(__WATCOMC__)) || defined(STRINGS_H_DECLARES_FFS) + /* + * MS-DOS with Watcom C, which has <strings.h> and declares ffs() there, + * or some other platform (UN*X conforming to a sufficient recent version + * of the Single UNIX Specification). + */ + #include <strings.h> + #define lowest_set_bit(mask) (ffs((mask)) - 1) +#else +/* + * None of the above. + * Use a perfect-hash-function-based function. */ static int -ffs(int mask) +lowest_set_bit(int mask) { - int bit; + unsigned int v = (unsigned int)mask; + + static const int MultiplyDeBruijnBitPosition[32] = { + 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 + }; - if (mask == 0) - return(0); - for (bit = 1; !(mask & 1); bit++) - mask >>= 1; - return(bit); + /* + * We strip off all but the lowermost set bit (v & ~v), + * and perform a minimal perfect hash on it to look up the + * number of low-order zero bits in a table. + * + * See: + * + * http://7ooo.mooo.com/text/ComputingTrailingZerosHOWTO.pdf + * + * http://supertech.csail.mit.edu/papers/debruijn.pdf + */ + return (MultiplyDeBruijnBitPosition[((v & -v) * 0x077CB531U) >> 27]); } #endif @@ -129,7 +227,7 @@ struct vmapinfo { bpf_int32 const_val; }; -struct _opt_state { +typedef struct { /* * A flag to indicate that further optimization is needed. * Iterative passes are continued until a given pass yields no @@ -212,7 +310,7 @@ struct _opt_state { struct vmapinfo *vmap; struct valnode *vnode_base; struct valnode *next_vnode; -}; +} opt_state_t; typedef struct { /* @@ -292,7 +390,7 @@ find_dom(opt_state_t *opt_state, struct block *root) x = opt_state->all_dom_sets; i = opt_state->n_blocks * opt_state->nodewords; while (--i >= 0) - *x++ = ~0; + *x++ = 0xFFFFFFFFU; /* Root starts off empty. */ for (i = opt_state->nodewords; --i >= 0;) root->dom[i] = 0; @@ -332,7 +430,7 @@ find_edom(opt_state_t *opt_state, struct block *root) x = opt_state->all_edge_sets; for (i = opt_state->n_edges * opt_state->edgewords; --i >= 0; ) - x[i] = ~0; + x[i] = 0xFFFFFFFFU; /* root->level is the highest level no found. */ memset(root->et.edom, 0, opt_state->edgewords * sizeof(*(uset)0)); @@ -592,7 +690,7 @@ F(opt_state_t *opt_state, int code, int v0, int v1) static inline void vstore(struct stmt *s, int *valp, int newval, int alter) { - if (alter && *valp == newval) + if (alter && newval != VAL_UNKNOWN && *valp == newval) s->code = NOP; else *valp = newval; @@ -603,7 +701,7 @@ vstore(struct stmt *s, int *valp, int newval, int alter) * (Unary operators are handled elsewhere.) */ static void -fold_op(compiler_state_t *cstate, struct icode *ic, opt_state_t *opt_state, +fold_op(compiler_state_t *cstate, opt_state_t *opt_state, struct stmt *s, int v0, int v1) { bpf_u_int32 a, b; @@ -945,7 +1043,7 @@ opt_peep(opt_state_t *opt_state, struct block *b) * evaluation and code transformations weren't folded together. */ static void -opt_stmt(compiler_state_t *cstate, struct icode *ic, opt_state_t *opt_state, +opt_stmt(compiler_state_t *cstate, opt_state_t *opt_state, struct stmt *s, int val[], int alter) { int op; @@ -1034,7 +1132,7 @@ opt_stmt(compiler_state_t *cstate, struct icode *ic, opt_state_t *opt_state, } } if (opt_state->vmap[val[A_ATOM]].is_const) { - fold_op(cstate, ic, opt_state, s, val[A_ATOM], K(s->k)); + fold_op(cstate, opt_state, s, val[A_ATOM], K(s->k)); val[A_ATOM] = K(s->k); break; } @@ -1055,7 +1153,7 @@ opt_stmt(compiler_state_t *cstate, struct icode *ic, opt_state_t *opt_state, op = BPF_OP(s->code); if (alter && opt_state->vmap[val[X_ATOM]].is_const) { if (opt_state->vmap[val[A_ATOM]].is_const) { - fold_op(cstate, ic, opt_state, s, val[A_ATOM], val[X_ATOM]); + fold_op(cstate, opt_state, s, val[A_ATOM], val[X_ATOM]); val[A_ATOM] = K(s->k); } else { @@ -1179,7 +1277,7 @@ opt_deadstores(opt_state_t *opt_state, register struct block *b) } static void -opt_blk(compiler_state_t *cstate, struct icode *ic, opt_state_t *opt_state, +opt_blk(compiler_state_t *cstate, opt_state_t *opt_state, struct block *b, int do_stmts) { struct slist *s; @@ -1230,7 +1328,7 @@ opt_blk(compiler_state_t *cstate, struct icode *ic, opt_state_t *opt_state, aval = b->val[A_ATOM]; xval = b->val[X_ATOM]; for (s = b->stmts; s; s = s->next) - opt_stmt(cstate, ic, opt_state, &s->s, b->val, do_stmts); + opt_stmt(cstate, opt_state, &s->s, b->val, do_stmts); /* * This is a special case: if we don't use anything from this @@ -1256,8 +1354,9 @@ opt_blk(compiler_state_t *cstate, struct icode *ic, opt_state_t *opt_state, * block, can we eliminate it? */ if (do_stmts && - ((b->out_use == 0 && aval != 0 && b->val[A_ATOM] == aval && - xval != 0 && b->val[X_ATOM] == xval) || + ((b->out_use == 0 && + aval != VAL_UNKNOWN && b->val[A_ATOM] == aval && + xval != VAL_UNKNOWN && b->val[X_ATOM] == xval) || BPF_CLASS(b->s.code) == BPF_RET)) { if (b->stmts != 0) { b->stmts = 0; @@ -1382,7 +1481,7 @@ opt_j(opt_state_t *opt_state, struct edge *ep) register bpf_u_int32 x = ep->edom[i]; while (x != 0) { - k = ffs(x) - 1; + k = lowest_set_bit(x); x &=~ (1 << k); k += i * BITS_PER_WORD; @@ -1433,7 +1532,7 @@ or_pullup(opt_state_t *opt_state, struct block *b) diffp = &JF(b->in_edges->pred); at_top = 1; - while (1) { + for (;;) { if (*diffp == 0) return; @@ -1450,7 +1549,7 @@ or_pullup(opt_state_t *opt_state, struct block *b) at_top = 0; } samep = &JF(*diffp); - while (1) { + for (;;) { if (*samep == 0) return; @@ -1524,7 +1623,7 @@ and_pullup(opt_state_t *opt_state, struct block *b) diffp = &JF(b->in_edges->pred); at_top = 1; - while (1) { + for (;;) { if (*diffp == 0) return; @@ -1541,7 +1640,7 @@ and_pullup(opt_state_t *opt_state, struct block *b) at_top = 0; } samep = &JT(*diffp); - while (1) { + for (;;) { if (*samep == 0) return; @@ -1602,7 +1701,7 @@ opt_blks(compiler_state_t *cstate, opt_state_t *opt_state, struct icode *ic, find_inedges(opt_state, ic->root); for (i = maxlevel; i >= 0; --i) for (p = opt_state->levels[i]; p; p = p->link) - opt_blk(cstate, ic, opt_state, p, do_stmts); + opt_blk(cstate, opt_state, p, do_stmts); if (do_stmts) /* @@ -1685,7 +1784,7 @@ opt_loop(compiler_state_t *cstate, opt_state_t *opt_state, struct icode *ic, { #ifdef BDEBUG - if (pcap_optimizer_debug > 1) { + if (pcap_optimizer_debug > 1 || pcap_print_dot_graph) { printf("opt_loop(root, %d) begin\n", do_stmts); opt_dump(cstate, ic); } @@ -1699,7 +1798,7 @@ opt_loop(compiler_state_t *cstate, opt_state_t *opt_state, struct icode *ic, find_edom(opt_state, ic->root); opt_blks(cstate, opt_state, ic, do_stmts); #ifdef BDEBUG - if (pcap_optimizer_debug > 1) { + if (pcap_optimizer_debug > 1 || pcap_print_dot_graph) { printf("opt_loop(root, %d) bottom, done=%d\n", do_stmts, opt_state->done); opt_dump(cstate, ic); } @@ -1720,14 +1819,14 @@ bpf_optimize(compiler_state_t *cstate, struct icode *ic) opt_loop(cstate, &opt_state, ic, 1); intern_blocks(&opt_state, ic); #ifdef BDEBUG - if (pcap_optimizer_debug > 1) { + if (pcap_optimizer_debug > 1 || pcap_print_dot_graph) { printf("after intern_blocks()\n"); opt_dump(cstate, ic); } #endif opt_root(&ic->root); #ifdef BDEBUG - if (pcap_optimizer_debug > 1) { + if (pcap_optimizer_debug > 1 || pcap_print_dot_graph) { printf("after opt_root()\n"); opt_dump(cstate, ic); } @@ -1765,7 +1864,7 @@ mark_code(struct icode *ic) static int eq_slist(struct slist *x, struct slist *y) { - while (1) { + for (;;) { while (x && x->s.code == NOP) x = x->next; while (y && y->s.code == NOP) @@ -2015,7 +2114,7 @@ opt_init(compiler_state_t *cstate, opt_state_t *opt_state, struct icode *ic) * and expect it to provide meaningful information. */ #ifdef BDEBUG -int bids[1000]; +int bids[NBIDS]; #endif /* @@ -2032,7 +2131,7 @@ convert_code_r(compiler_state_t *cstate, conv_state_t *conv_state, struct slist *src; u_int slen; u_int off; - int extrajmps; /* number of extra jumps inserted */ + u_int extrajmps; /* number of extra jumps inserted */ struct slist **offset = NULL; if (p == 0 || isMarked(ic, p)) @@ -2090,7 +2189,7 @@ convert_code_r(compiler_state_t *cstate, conv_state_t *conv_state, { u_int i; int jt, jf; - const char *ljerr = "%s for block-local relative jump: off=%d"; + const char ljerr[] = "%s for block-local relative jump: off=%d"; #if 0 printf("code=%x off=%d %x %x\n", src->s.code, @@ -2110,7 +2209,11 @@ convert_code_r(compiler_state_t *cstate, conv_state_t *conv_state, /*NOTREACHED*/ } - dst->jt = i - off - 1; + if (i - off - 1 >= 256) { + bpf_error(cstate, ljerr, "out-of-range jump", off); + /*NOTREACHED*/ + } + dst->jt = (u_char)(i - off - 1); jt++; } if (offset[i] == src->s.jf) { @@ -2118,7 +2221,11 @@ convert_code_r(compiler_state_t *cstate, conv_state_t *conv_state, bpf_error(cstate, ljerr, "multiple matches", off); /*NOTREACHED*/ } - dst->jf = i - off - 1; + if (i - off - 1 >= 256) { + bpf_error(cstate, ljerr, "out-of-range jump", off); + /*NOTREACHED*/ + } + dst->jf = (u_char)(i - off - 1); jf++; } } @@ -2135,7 +2242,8 @@ filled: free(offset); #ifdef BDEBUG - bids[dst - conv_state->fstart] = p->id + 1; + if (dst - conv_state->fstart < NBIDS) + bids[dst - conv_state->fstart] = p->id + 1; #endif dst->code = (u_short)p->s.code; dst->k = p->s.k; @@ -2150,13 +2258,17 @@ filled: return(0); } /* branch if T to following jump */ - dst->jt = extrajmps; + if (extrajmps >= 256) { + bpf_error(cstate, "too many extra jumps"); + /*NOTREACHED*/ + } + dst->jt = (u_char)extrajmps; extrajmps++; dst[extrajmps].code = BPF_JMP|BPF_JA; dst[extrajmps].k = off - extrajmps; } else - dst->jt = off; + dst->jt = (u_char)off; off = JF(p)->offset - (p->offset + slen) - 1; if (off >= 256) { /* offset too large for branch, must add a jump */ @@ -2167,13 +2279,17 @@ filled: } /* branch if F to following jump */ /* if two jumps are inserted, F goes to second one */ - dst->jf = extrajmps; + if (extrajmps >= 256) { + bpf_error(cstate, "too many extra jumps"); + /*NOTREACHED*/ + } + dst->jf = (u_char)extrajmps; extrajmps++; dst[extrajmps].code = BPF_JMP|BPF_JA; dst[extrajmps].k = off - extrajmps; } else - dst->jf = off; + dst->jf = (u_char)off; } return (1); } @@ -2209,7 +2325,7 @@ icode_to_fcode(compiler_state_t *cstate, struct icode *ic, * Loop doing convert_code_r() until no branches remain * with too-large offsets. */ - while (1) { + for (;;) { unMarkAll(ic); n = *lenp = count_stmts(ic, root); @@ -2260,8 +2376,8 @@ install_bpf_program(pcap_t *p, struct bpf_program *fp) p->fcode.bf_len = fp->bf_len; p->fcode.bf_insns = (struct bpf_insn *)malloc(prog_size); if (p->fcode.bf_insns == NULL) { - pcap_snprintf(p->errbuf, sizeof(p->errbuf), - "malloc: %s", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, sizeof(p->errbuf), + errno, "malloc"); return (-1); } memcpy(p->fcode.bf_insns, fp->bf_insns, prog_size); @@ -2289,7 +2405,7 @@ dot_dump_node(struct icode *ic, struct block *block, struct bpf_program *prog, } fprintf(out, "\" tooltip=\""); for (i = 0; i < BPF_MEMWORDS; i++) - if (block->val[i] != 0) + if (block->val[i] != VAL_UNKNOWN) fprintf(out, "val[%d]=%d ", i, block->val[i]); fprintf(out, "val[A]=%d ", block->val[A_ATOM]); fprintf(out, "val[X]=%d", block->val[X_ATOM]); @@ -2348,10 +2464,8 @@ dot_dump(compiler_state_t *cstate, struct icode *ic) f.bf_insns = icode_to_fcode(cstate, ic, ic->root, &f.bf_len); fprintf(out, "digraph BPF {\n"); - ic->cur_mark = 0; unMarkAll(ic); dot_dump_node(ic, ic->root, &f, out); - ic->cur_mark = 0; unMarkAll(ic); dot_dump_edge(ic, ic->root, out); fprintf(out, "}\n"); @@ -2374,11 +2488,11 @@ plain_dump(compiler_state_t *cstate, struct icode *ic) static void opt_dump(compiler_state_t *cstate, struct icode *ic) { - /* if optimizer debugging is enabled, output DOT graph - * `pcap_optimizer_debug=4' is equivalent to -dddd to follow -d/-dd/-ddd - * convention in tcpdump command line + /* + * If the CFG, in DOT format, is requested, output it rather than + * the code that would be generated from that graph. */ - if (pcap_optimizer_debug > 3) + if (pcap_print_dot_graph) dot_dump(cstate, ic); else plain_dump(cstate, ic); diff --git a/freebsd/contrib/libpcap/optimize.h b/freebsd/contrib/libpcap/optimize.h new file mode 100644 index 00000000..56b31f40 --- /dev/null +++ b/freebsd/contrib/libpcap/optimize.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 1990, 1991, 1992, 1993, 1994, 1995, 1996 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that: (1) source code distributions + * retain the above copyright notice and this paragraph in its entirety, (2) + * distributions including binary code include the above copyright notice and + * this paragraph in its entirety in the documentation or other materials + * provided with the distribution, and (3) all advertising materials mentioning + * features or use of this software display the following acknowledgement: + * ``This product includes software developed by the University of California, + * Lawrence Berkeley Laboratory and its contributors.'' Neither the name of + * the University nor the names of its contributors may be used to endorse + * or promote products derived from this software without specific prior + * written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +/* + * Some stuff for use when debugging the optimizer. + */ +#ifdef BDEBUG +#define NBIDS 1000 +extern int bids[NBIDS]; +#endif diff --git a/freebsd/contrib/libpcap/pcap-bpf.c b/freebsd/contrib/libpcap/pcap-bpf.c index 60111cf5..dc2942af 100644 --- a/freebsd/contrib/libpcap/pcap-bpf.c +++ b/freebsd/contrib/libpcap/pcap-bpf.c @@ -19,25 +19,20 @@ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. - * - * $FreeBSD$ */ #ifdef HAVE_CONFIG_H -#include "config.h" +#include <config.h> #endif #include <sys/param.h> /* optionally get BSD define */ -#ifdef HAVE_ZEROCOPY_BPF -#include <sys/mman.h> -#endif #include <sys/socket.h> #include <time.h> /* * <net/bpf.h> defines ioctls, but doesn't include <sys/ioccom.h>. * * We include <sys/ioctl.h> as it might be necessary to declare ioctl(); - * at least on *BSD and Mac OS X, it also defines various SIOC ioctls - + * at least on *BSD and macOS, it also defines various SIOC ioctls - * we could include <sys/sockio.h>, but if we're already including * <sys/ioctl.h>, which includes <sys/sockio.h> on those platforms, * there's not much point in doing so. @@ -61,10 +56,6 @@ static const char usbus_prefix[] = "usbus"; #include <dirent.h> #endif -#ifdef HAVE_ZEROCOPY_BPF -#include <machine/atomic.h> -#endif - #include <net/if.h> #ifdef _AIX @@ -86,6 +77,16 @@ static const char usbus_prefix[] = "usbus"; #include <net/bpf.h> #define _AIX +/* + * If both BIOCROTZBUF and BPF_BUFMODE_ZBUF are defined, we have + * zero-copy BPF. + */ +#if defined(BIOCROTZBUF) && defined(BPF_BUFMODE_ZBUF) + #define HAVE_ZEROCOPY_BPF + #include <sys/mman.h> + #include <machine/atomic.h> +#endif + #include <net/if_types.h> /* for IFT_ values */ #include <sys/sysconfig.h> #include <sys/device.h> @@ -125,7 +126,7 @@ static int bpf_load(char *errbuf); #include <string.h> #include <unistd.h> -#ifdef HAVE_NET_IF_MEDIA_H +#ifdef SIOCGIFMEDIA # include <net/if_media.h> #endif @@ -228,8 +229,9 @@ static void remove_802_11(pcap_t *); #endif /* - * On OS X, we don't even get any of the 802.11-plus-radio-header DLT_'s - * defined, even though some of them are used by various Airport drivers. + * In some versions of macOS, we might not even get any of the + * 802.11-plus-radio-header DLT_'s defined, even though some + * of them are used by various Airport drivers in those versions. */ #ifndef DLT_PRISM_HEADER #define DLT_PRISM_HEADER 119 @@ -256,7 +258,7 @@ static int pcap_set_datalink_bpf(pcap_t *p, int dlt); * blocking mode. */ static int -pcap_getnonblock_bpf(pcap_t *p, char *errbuf) +pcap_getnonblock_bpf(pcap_t *p) { #ifdef HAVE_ZEROCOPY_BPF struct pcap_bpf *pb = p->priv; @@ -264,11 +266,11 @@ pcap_getnonblock_bpf(pcap_t *p, char *errbuf) if (pb->zerocopy) return (pb->nonblock); #endif - return (pcap_getnonblock_fd(p, errbuf)); + return (pcap_getnonblock_fd(p)); } static int -pcap_setnonblock_bpf(pcap_t *p, int nonblock, char *errbuf) +pcap_setnonblock_bpf(pcap_t *p, int nonblock) { #ifdef HAVE_ZEROCOPY_BPF struct pcap_bpf *pb = p->priv; @@ -278,7 +280,7 @@ pcap_setnonblock_bpf(pcap_t *p, int nonblock, char *errbuf) return (0); } #endif - return (pcap_setnonblock_fd(p, nonblock, errbuf)); + return (pcap_setnonblock_fd(p, nonblock)); } #ifdef HAVE_ZEROCOPY_BPF @@ -369,8 +371,8 @@ pcap_next_zbuf(pcap_t *p, int *cc) if (data) return (data); if (ioctl(p->fd, BIOCROTZBUF, &bz) < 0) { - (void) pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "BIOCROTZBUF: %s", strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, + PCAP_ERRBUF_SIZE, errno, "BIOCROTZBUF"); return (PCAP_ERROR); } return (pcap_next_zbuf_shm(p, cc)); @@ -397,8 +399,8 @@ pcap_next_zbuf(pcap_t *p, int *cc) } return (0); } else if (r < 0) { - (void) pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "select: %s", strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "select"); return (PCAP_ERROR); } } @@ -416,8 +418,8 @@ pcap_next_zbuf(pcap_t *p, int *cc) * data. */ if (ioctl(p->fd, BIOCROTZBUF, &bz) < 0) { - (void) pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "BIOCROTZBUF: %s", strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "BIOCROTZBUF"); return (PCAP_ERROR); } return (pcap_next_zbuf_shm(p, cc)); @@ -459,8 +461,8 @@ pcap_create_interface(const char *device _U_, char *ebuf) p->tstamp_precision_count = 2; p->tstamp_precision_list = malloc(2 * sizeof(u_int)); if (p->tstamp_precision_list == NULL) { - snprintf(ebuf, PCAP_ERRBUF_SIZE, "malloc: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(ebuf, PCAP_ERRBUF_SIZE, errno, + "malloc"); free(p); return (NULL); } @@ -477,13 +479,11 @@ pcap_create_interface(const char *device _U_, char *ebuf) static int bpf_open(char *errbuf) { - int fd; -#ifdef HAVE_CLONING_BPF - static const char device[] = "/dev/bpf"; -#else + int fd = -1; + static const char cloning_device[] = "/dev/bpf"; int n = 0; char device[sizeof "/dev/bpf0000000000"]; -#endif + static int no_cloning_bpf = 0; #ifdef _AIX /* @@ -495,40 +495,55 @@ bpf_open(char *errbuf) return (PCAP_ERROR); #endif -#ifdef HAVE_CLONING_BPF - if ((fd = open(device, O_RDWR)) == -1 && - (errno != EACCES || (fd = open(device, O_RDONLY)) == -1)) { - if (errno == EACCES) - fd = PCAP_ERROR_PERM_DENIED; - else - fd = PCAP_ERROR; - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "(cannot open device) %s: %s", device, pcap_strerror(errno)); - } -#else /* - * Go through all the minors and find one that isn't in use. + * First, unless we've already tried opening /dev/bpf and + * gotten ENOENT, try opening /dev/bpf. + * If it fails with ENOENT, remember that, so we don't try + * again, and try /dev/bpfN. */ - do { - (void)pcap_snprintf(device, sizeof(device), "/dev/bpf%d", n++); + if (!no_cloning_bpf && + (fd = open(cloning_device, O_RDWR)) == -1 && + ((errno != EACCES && errno != ENOENT) || + (fd = open(cloning_device, O_RDONLY)) == -1)) { + if (errno != ENOENT) { + if (errno == EACCES) + fd = PCAP_ERROR_PERM_DENIED; + else + fd = PCAP_ERROR; + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "(cannot open device) %s", cloning_device); + return (fd); + } + no_cloning_bpf = 1; + } + + if (no_cloning_bpf) { /* - * Initially try a read/write open (to allow the inject - * method to work). If that fails due to permission - * issues, fall back to read-only. This allows a - * non-root user to be granted specific access to pcap - * capabilities via file permissions. - * - * XXX - we should have an API that has a flag that - * controls whether to open read-only or read-write, - * so that denial of permission to send (or inability - * to send, if sending packets isn't supported on - * the device in question) can be indicated at open - * time. + * We don't have /dev/bpf. + * Go through all the /dev/bpfN minors and find one + * that isn't in use. */ - fd = open(device, O_RDWR); - if (fd == -1 && errno == EACCES) - fd = open(device, O_RDONLY); - } while (fd < 0 && errno == EBUSY); + do { + (void)pcap_snprintf(device, sizeof(device), "/dev/bpf%d", n++); + /* + * Initially try a read/write open (to allow the inject + * method to work). If that fails due to permission + * issues, fall back to read-only. This allows a + * non-root user to be granted specific access to pcap + * capabilities via file permissions. + * + * XXX - we should have an API that has a flag that + * controls whether to open read-only or read-write, + * so that denial of permission to send (or inability + * to send, if sending packets isn't supported on + * the device in question) can be indicated at open + * time. + */ + fd = open(device, O_RDWR); + if (fd == -1 && errno == EACCES) + fd = open(device, O_RDONLY); + } while (fd < 0 && errno == EBUSY); + } /* * XXX better message for all minors used @@ -565,9 +580,8 @@ bpf_open(char *errbuf) * if any. */ fd = PCAP_ERROR_PERM_DENIED; - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "(cannot open BPF device) %s: %s", device, - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "(cannot open BPF device) %s", device); break; default: @@ -575,13 +589,11 @@ bpf_open(char *errbuf) * Some other problem. */ fd = PCAP_ERROR; - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "(cannot open BPF device) %s: %s", device, - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "(cannot open BPF device) %s", device); break; } } -#endif return (fd); } @@ -633,8 +645,8 @@ bpf_open_and_bind(const char *name, char *errbuf) return (PCAP_ERROR_IFACE_NOT_UP); default: - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "BIOCSETIF: %s: %s", name, pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "BIOCSETIF: %s", name); close(fd); return (PCAP_ERROR); } @@ -657,14 +669,14 @@ get_dlt_list(int fd, int v, struct bpf_dltlist *bdlp, char *ebuf) bdlp->bfl_list = (u_int *) malloc(sizeof(u_int) * (bdlp->bfl_len + 1)); if (bdlp->bfl_list == NULL) { - (void)pcap_snprintf(ebuf, PCAP_ERRBUF_SIZE, "malloc: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(ebuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); return (PCAP_ERROR); } if (ioctl(fd, BIOCGDLTLIST, (caddr_t)bdlp) < 0) { - (void)pcap_snprintf(ebuf, PCAP_ERRBUF_SIZE, - "BIOCGDLTLIST: %s", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(ebuf, PCAP_ERRBUF_SIZE, + errno, "BIOCGDLTLIST"); free(bdlp->bfl_list); return (PCAP_ERROR); } @@ -718,8 +730,8 @@ get_dlt_list(int fd, int v, struct bpf_dltlist *bdlp, char *ebuf) * this device"; don't treat it as an error. */ if (errno != EINVAL) { - (void)pcap_snprintf(ebuf, PCAP_ERRBUF_SIZE, - "BIOCGDLTLIST: %s", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(ebuf, PCAP_ERRBUF_SIZE, + errno, "BIOCGDLTLIST"); return (PCAP_ERROR); } } @@ -739,7 +751,7 @@ pcap_can_set_rfmon_bpf(pcap_t *p) #endif /* - * The joys of monitor mode on OS X. + * The joys of monitor mode on Mac OS X/OS X/macOS. * * Prior to 10.4, it's not supported at all. * @@ -785,8 +797,8 @@ pcap_can_set_rfmon_bpf(pcap_t *p) } fd = socket(AF_INET, SOCK_DGRAM, 0); if (fd == -1) { - (void)pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "socket: %s", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "socket"); return (PCAP_ERROR); } strlcpy(ifr.ifr_name, "wlt", sizeof(ifr.ifr_name)); @@ -840,9 +852,8 @@ pcap_can_set_rfmon_bpf(pcap_t *p) return (PCAP_ERROR_IFACE_NOT_UP); default: - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "BIOCSETIF: %s: %s", - p->opt.device, pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "BIOCSETIF: %s", p->opt.device); close(fd); return (PCAP_ERROR); } @@ -904,8 +915,8 @@ pcap_stats_bpf(pcap_t *p, struct pcap_stat *ps) * by libpcap, and thus not yet seen by the application. */ if (ioctl(p->fd, BIOCGSTATS, (caddr_t)&s) < 0) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "BIOCGSTATS: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "BIOCGSTATS"); return (PCAP_ERROR); } @@ -1032,8 +1043,8 @@ pcap_read_bpf(pcap_t *p, int cnt, pcap_handler callback, u_char *user) /* fall through */ #endif } - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "read: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "read"); return (PCAP_ERROR); } bp = (u_char *)p->buffer; @@ -1186,13 +1197,13 @@ pcap_inject_bpf(pcap_t *p, const void *buf, size_t size) #ifdef __APPLE__ if (ret == -1 && errno == EAFNOSUPPORT) { /* - * In Mac OS X, there's a bug wherein setting the - * BIOCSHDRCMPLT flag causes writes to fail; see, - * for example: + * In some versions of macOS, there's a bug wherein setting + * the BIOCSHDRCMPLT flag causes writes to fail; see, for + * example: * * http://cerberus.sourcefire.com/~jeff/archives/patches/macosx/BIOCSHDRCMPLT-10.3.3.patch * - * So, if, on OS X, we get EAFNOSUPPORT from the write, we + * So, if, on macOS, we get EAFNOSUPPORT from the write, we * assume it's due to that bug, and turn off that flag * and try again. If we succeed, it either means that * somebody applied the fix from that URL, or other patches @@ -1201,14 +1212,13 @@ pcap_inject_bpf(pcap_t *p, const void *buf, size_t size) * http://cerberus.sourcefire.com/~jeff/archives/patches/macosx/ * * and are running a Darwin kernel with those fixes, or - * that Apple fixed the problem in some OS X release. + * that Apple fixed the problem in some macOS release. */ u_int spoof_eth_src = 0; if (ioctl(p->fd, BIOCSHDRCMPLT, &spoof_eth_src) == -1) { - (void)pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "send: can't turn off BIOCSHDRCMPLT: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "send: can't turn off BIOCSHDRCMPLT"); return (PCAP_ERROR); } @@ -1219,8 +1229,8 @@ pcap_inject_bpf(pcap_t *p, const void *buf, size_t size) } #endif /* __APPLE__ */ if (ret == -1) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "send: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "send"); return (PCAP_ERROR); } return (ret); @@ -1308,8 +1318,8 @@ bpf_load(char *errbuf) major = genmajor(BPF_NAME); if (major == -1) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "bpf_load: genmajor failed: %s", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "bpf_load: genmajor failed"); (void)bpf_odmcleanup(NULL); return (PCAP_ERROR); } @@ -1318,9 +1328,8 @@ bpf_load(char *errbuf) if (!minors) { minors = genminor("bpf", major, 0, BPF_MINORS, 1, 1); if (!minors) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "bpf_load: genminor failed: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "bpf_load: genminor failed"); (void)bpf_odmcleanup(NULL); return (PCAP_ERROR); } @@ -1331,20 +1340,19 @@ bpf_load(char *errbuf) rc = stat(BPF_NODE "0", &sbuf); if (rc == -1 && errno != ENOENT) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "bpf_load: can't stat %s: %s", - BPF_NODE "0", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "bpf_load: can't stat %s", BPF_NODE "0"); return (PCAP_ERROR); } if (rc == -1 || getmajor(sbuf.st_rdev) != major) { for (i = 0; i < BPF_MINORS; i++) { - sprintf(buf, "%s%d", BPF_NODE, i); + pcap_snprintf(buf, sizeof(buf), "%s%d", BPF_NODE, i); unlink(buf); if (mknod(buf, S_IRUSR | S_IFCHR, domakedev(major, i)) == -1) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "bpf_load: can't mknod %s: %s", - buf, pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, + PCAP_ERRBUF_SIZE, errno, + "bpf_load: can't mknod %s", buf); return (PCAP_ERROR); } } @@ -1353,14 +1361,13 @@ bpf_load(char *errbuf) /* Check if the driver is loaded */ memset(&cfg_ld, 0x0, sizeof(cfg_ld)); cfg_ld.path = buf; - sprintf(cfg_ld.path, "%s/%s", DRIVER_PATH, BPF_NAME); + pcap_snprintf(cfg_ld.path, sizeof(cfg_ld.path), "%s/%s", DRIVER_PATH, BPF_NAME); if ((sysconfig(SYS_QUERYLOAD, (void *)&cfg_ld, sizeof(cfg_ld)) == -1) || (cfg_ld.kmid == 0)) { /* Driver isn't loaded, load it now */ if (sysconfig(SYS_SINGLELOAD, (void *)&cfg_ld, sizeof(cfg_ld)) == -1) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "bpf_load: could not load driver: %s", - strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "bpf_load: could not load driver"); return (PCAP_ERROR); } } @@ -1373,9 +1380,8 @@ bpf_load(char *errbuf) for (i = 0; i < BPF_MINORS; i++) { cfg_bpf.devno = domakedev(major, i); if (sysconfig(SYS_CFGKMOD, (void *)&cfg_km, sizeof(cfg_km)) == -1) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "bpf_load: could not configure driver: %s", - strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "bpf_load: could not configure driver"); return (PCAP_ERROR); } } @@ -1495,7 +1501,6 @@ pcap_cleanup_bpf(pcap_t *p) if (pb->zbuf2 != MAP_FAILED && pb->zbuf2 != NULL) (void) munmap(pb->zbuf2, pb->zbufsize); p->buffer = NULL; - p->buffer = NULL; } #endif if (pb->device != NULL) { @@ -1540,9 +1545,10 @@ check_setif_failure(pcap_t *p, int error) * exist. */ err = PCAP_ERROR_NO_SUCH_DEVICE; - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "SIOCGIFFLAGS on %s failed: %s", - ifr.ifr_name, pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, + PCAP_ERRBUF_SIZE, errno, + "SIOCGIFFLAGS on %s failed", + ifr.ifr_name); } else { /* * The underlying "enN" device @@ -1564,9 +1570,9 @@ check_setif_failure(pcap_t *p, int error) * just report "no such device". */ err = PCAP_ERROR_NO_SUCH_DEVICE; - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "socket() failed: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, + errno, PCAP_ERRBUF_SIZE, + "socket() failed"); } return (err); } @@ -1574,8 +1580,8 @@ check_setif_failure(pcap_t *p, int error) /* * No such device. */ - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "BIOCSETIF failed: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "BIOCSETIF failed"); return (PCAP_ERROR_NO_SUCH_DEVICE); } else if (errno == ENETDOWN) { /* @@ -1591,8 +1597,8 @@ check_setif_failure(pcap_t *p, int error) * Some other error; fill in the error string, and * return PCAP_ERROR. */ - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "BIOCSETIF: %s: %s", - p->opt.device, pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "BIOCSETIF: %s", p->opt.device); return (PCAP_ERROR); } } @@ -1649,15 +1655,6 @@ pcap_activate_bpf(pcap_t *p) struct bpf_insn total_insn; struct bpf_program total_prog; struct utsname osinfo; - -#ifdef HAVE_DAG_API - if (strstr(device, "dag")) { - return dag_open_live(device, snaplen, promisc, to_ms, ebuf); - } -#endif /* HAVE_DAG_API */ - -#ifdef BIOCGDLTLIST - memset(&bdl, 0, sizeof(bdl)); int have_osinfo = 0; #ifdef HAVE_ZEROCOPY_BPF struct bpf_zbuf bz; @@ -1673,8 +1670,8 @@ pcap_activate_bpf(pcap_t *p) p->fd = fd; if (ioctl(fd, BIOCVERSION, (caddr_t)&bv) < 0) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "BIOCVERSION: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "BIOCVERSION"); status = PCAP_ERROR; goto bad; } @@ -1686,13 +1683,24 @@ pcap_activate_bpf(pcap_t *p) goto bad; } + /* + * Turn a negative snapshot value (invalid), a snapshot value of + * 0 (unspecified), or a value bigger than the normal maximum + * value, into the maximum allowed value. + * + * If some application really *needs* a bigger snapshot + * length, we should just increase MAXIMUM_SNAPLEN. + */ + if (p->snapshot <= 0 || p->snapshot > MAXIMUM_SNAPLEN) + p->snapshot = MAXIMUM_SNAPLEN; + #if defined(LIFNAMSIZ) && defined(ZONENAME_MAX) && defined(lifr_zoneid) /* * Retrieve the zoneid of the zone we are currently executing in. */ if ((ifr.lifr_zoneid = getzoneid()) == -1) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "getzoneid(): %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "getzoneid()"); status = PCAP_ERROR; goto bad; } @@ -1718,16 +1726,15 @@ pcap_activate_bpf(pcap_t *p) (void) strlcpy(path_zname, p->opt.device, znamelen + 1); ifr.lifr_zoneid = getzoneidbyname(path_zname); if (ifr.lifr_zoneid == -1) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "getzoneidbyname(%s): %s", path_zname, - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "getzoneidbyname(%s)", path_zname); status = PCAP_ERROR; goto bad; } lnamep = strdup(zonesep + 1); if (lnamep == NULL) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "strdup: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "strdup"); status = PCAP_ERROR; goto bad; } @@ -1738,20 +1745,13 @@ pcap_activate_bpf(pcap_t *p) pb->device = strdup(p->opt.device); if (pb->device == NULL) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "strdup: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "strdup"); status = PCAP_ERROR; goto bad; } /* - * Try finding a good size for the buffer; 32768 may be too - * big, so keep cutting it in half until we find a size - * that works, or run out of sizes to try. If the default - * is larger, don't make it smaller. - * - * XXX - there should be a user-accessible hook to set the - * initial buffer size. * Attempt to find out the version of the OS on which we're running. */ if (uname(&osinfo) == 0) @@ -1800,10 +1800,10 @@ pcap_activate_bpf(pcap_t *p) * exist. */ status = PCAP_ERROR_NO_SUCH_DEVICE; - pcap_snprintf(p->errbuf, + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, - "SIOCGIFFLAGS failed: %s", - pcap_strerror(errno)); + errno, + "SIOCGIFFLAGS failed"); } else status = PCAP_ERROR_RFMON_NOTSUP; close(sockfd); @@ -1814,18 +1814,17 @@ pcap_activate_bpf(pcap_t *p) * report "no such device". */ status = PCAP_ERROR_NO_SUCH_DEVICE; - pcap_snprintf(p->errbuf, - PCAP_ERRBUF_SIZE, - "socket() failed: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, + PCAP_ERRBUF_SIZE, errno, + "socket() failed"); } goto bad; } wltdev = malloc(strlen(p->opt.device) + 2); if (wltdev == NULL) { - (void)pcap_snprintf(p->errbuf, - PCAP_ERRBUF_SIZE, "malloc: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, + PCAP_ERRBUF_SIZE, errno, + "malloc"); status = PCAP_ERROR; goto bad; } @@ -1865,9 +1864,9 @@ pcap_activate_bpf(pcap_t *p) */ s = socket(AF_LOCAL, SOCK_DGRAM, 0); if (s < 0) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "Can't open socket: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, + PCAP_ERRBUF_SIZE, errno, + "Can't open socket"); status = PCAP_ERROR; goto bad; } @@ -1898,9 +1897,10 @@ pcap_activate_bpf(pcap_t *p) "Invalid USB bus interface %s", p->opt.device); } else { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "Can't create interface for %s: %s", - p->opt.device, pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, + PCAP_ERRBUF_SIZE, errno, + "Can't create interface for %s", + p->opt.device); } close(s); status = PCAP_ERROR; @@ -1943,8 +1943,8 @@ pcap_activate_bpf(pcap_t *p) * size. */ if (ioctl(fd, BIOCGETZMAX, (caddr_t)&zbufmax) < 0) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "BIOCGETZMAX: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "BIOCGETZMAX"); status = PCAP_ERROR; goto bad; } @@ -1970,8 +1970,8 @@ pcap_activate_bpf(pcap_t *p) pb->zbuf2 = mmap(NULL, pb->zbufsize, PROT_READ | PROT_WRITE, MAP_ANON, -1, 0); if (pb->zbuf1 == MAP_FAILED || pb->zbuf2 == MAP_FAILED) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "mmap: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "mmap"); status = PCAP_ERROR; goto bad; } @@ -1980,15 +1980,15 @@ pcap_activate_bpf(pcap_t *p) bz.bz_bufb = pb->zbuf2; bz.bz_buflen = pb->zbufsize; if (ioctl(fd, BIOCSETZBUF, (caddr_t)&bz) < 0) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "BIOCSETZBUF: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "BIOCSETZBUF"); status = PCAP_ERROR; goto bad; } (void)strncpy(ifrname, p->opt.device, ifnamsiz); if (ioctl(fd, BIOCSETIF, (caddr_t)&ifr) < 0) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "BIOCSETIF: %s: %s", - p->opt.device, pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "BIOCSETIF: %s", p->opt.device); status = PCAP_ERROR; goto bad; } @@ -2006,9 +2006,9 @@ pcap_activate_bpf(pcap_t *p) */ if (ioctl(fd, BIOCSBLEN, (caddr_t)&p->opt.buffer_size) < 0) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "BIOCSBLEN: %s: %s", p->opt.device, - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, + PCAP_ERRBUF_SIZE, errno, + "BIOCSBLEN: %s", p->opt.device); status = PCAP_ERROR; goto bad; } @@ -2072,12 +2072,11 @@ pcap_activate_bpf(pcap_t *p) } } } -#endif /* Get the data link layer type. */ if (ioctl(fd, BIOCGDLT, (caddr_t)&v) < 0) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "BIOCGDLT: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "BIOCGDLT"); status = PCAP_ERROR; goto bad; } @@ -2321,8 +2320,8 @@ pcap_activate_bpf(pcap_t *p) * BSDs - check CVS log for "bpf.c"? */ if (ioctl(fd, BIOCSHDRCMPLT, &spoof_eth_src) == -1) { - (void)pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "BIOCSHDRCMPLT: %s", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "BIOCSHDRCMPLT"); status = PCAP_ERROR; goto bad; } @@ -2363,8 +2362,8 @@ pcap_activate_bpf(pcap_t *p) bpf_to.tv_sec = p->opt.timeout / 1000; bpf_to.tv_usec = (p->opt.timeout * 1000) % 1000000; if (ioctl(p->fd, BIOCSRTIMEOUT, (caddr_t)&bpf_to) < 0) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "BIOCSRTIMEOUT: %s", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, + errno, PCAP_ERRBUF_SIZE, "BIOCSRTIMEOUT"); status = PCAP_ERROR; goto bad; } @@ -2373,8 +2372,8 @@ pcap_activate_bpf(pcap_t *p) to.tv_sec = p->opt.timeout / 1000; to.tv_usec = (p->opt.timeout * 1000) % 1000000; if (ioctl(p->fd, BIOCSRTIMEOUT, (caddr_t)&to) < 0) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "BIOCSRTIMEOUT: %s", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, + errno, PCAP_ERRBUF_SIZE, "BIOCSRTIMEOUT"); status = PCAP_ERROR; goto bad; } @@ -2408,8 +2407,8 @@ pcap_activate_bpf(pcap_t *p) #endif /* _AIX */ v = 1; if (ioctl(p->fd, BIOCIMMEDIATE, &v) < 0) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "BIOCIMMEDIATE: %s", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "BIOCIMMEDIATE"); status = PCAP_ERROR; goto bad; } @@ -2430,8 +2429,8 @@ pcap_activate_bpf(pcap_t *p) if (p->opt.promisc) { /* set promiscuous mode, just warn if it fails */ if (ioctl(p->fd, BIOCPROMISC, NULL) < 0) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "BIOCPROMISC: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "BIOCPROMISC"); status = PCAP_WARNING_PROMISC_NOTSUP; } } @@ -2439,16 +2438,16 @@ pcap_activate_bpf(pcap_t *p) #ifdef BIOCSTSTAMP v = BPF_T_BINTIME; if (ioctl(p->fd, BIOCSTSTAMP, &v) < 0) { - snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "BIOCSTSTAMP: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "BIOCSTSTAMP"); status = PCAP_ERROR; goto bad; } #endif /* BIOCSTSTAMP */ if (ioctl(fd, BIOCGBLEN, (caddr_t)&v) < 0) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "BIOCGBLEN: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "BIOCGBLEN"); status = PCAP_ERROR; goto bad; } @@ -2458,8 +2457,8 @@ pcap_activate_bpf(pcap_t *p) #endif p->buffer = malloc(p->bufsize); if (p->buffer == NULL) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "malloc: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); status = PCAP_ERROR; goto bad; } @@ -2489,8 +2488,8 @@ pcap_activate_bpf(pcap_t *p) total_prog.bf_len = 1; total_prog.bf_insns = &total_insn; if (ioctl(p->fd, BIOCSETF, (caddr_t)&total_prog) < 0) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "BIOCSETF: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "BIOCSETF"); status = PCAP_ERROR; goto bad; } @@ -2572,6 +2571,44 @@ check_bpf_bindable(const char *name) int fd; char errbuf[PCAP_ERRBUF_SIZE]; + /* + * On macOS, we don't do this check if the device name begins + * with "wlt"; at least some versions of macOS (actually, it + * was called "Mac OS X" then...) offer monitor mode capturing + * by having a separate "monitor mode" device for each wireless + * adapter, rather than by implementing the ioctls that + * {Free,Net,Open,DragonFly}BSD provide. Opening that device + * puts the adapter into monitor mode, which, at least for + * some adapters, causes them to deassociate from the network + * with which they're associated. + * + * Instead, we try to open the corresponding "en" device (so + * that we don't end up with, for users without sufficient + * privilege to open capture devices, a list of adapters that + * only includes the wlt devices). + */ +#ifdef __APPLE__ + if (strncmp(name, "wlt", 3) == 0) { + char *en_name; + size_t en_name_len; + + /* + * Try to allocate a buffer for the "en" + * device's name. + */ + en_name_len = strlen(name) - 1; + en_name = malloc(en_name_len + 1); + if (en_name == NULL) { + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); + return (-1); + } + strcpy(en_name, "en"); + strcat(en_name, name + 3); + fd = bpf_open_and_bind(en_name, errbuf); + free(en_name); + } else +#endif /* __APPLE */ fd = bpf_open_and_bind(name, errbuf); if (fd < 0) { /* @@ -2606,7 +2643,18 @@ check_bpf_bindable(const char *name) #if defined(__FreeBSD__) && defined(SIOCIFCREATE2) static int -finddevs_usb(pcap_if_t **alldevsp, char *errbuf) +get_usb_if_flags(const char *name _U_, bpf_u_int32 *flags _U_, char *errbuf _U_) +{ + /* + * XXX - if there's a way to determine whether there's something + * plugged into a given USB bus, use that to determine whether + * this device is "connected" or not. + */ + return (0); +} + +static int +finddevs_usb(pcap_if_list_t *devlistp, char *errbuf) { DIR *usbdir; struct dirent *usbitem; @@ -2646,7 +2694,6 @@ finddevs_usb(pcap_if_t **alldevsp, char *errbuf) while ((usbitem = readdir(usbdir)) != NULL) { char *p; size_t busnumlen; - int err; if (strcmp(usbitem->d_name, ".") == 0 || strcmp(usbitem->d_name, "..") == 0) { @@ -2662,11 +2709,18 @@ finddevs_usb(pcap_if_t **alldevsp, char *errbuf) memcpy(name, usbus_prefix, USBUS_PREFIX_LEN); memcpy(name + USBUS_PREFIX_LEN, usbitem->d_name, busnumlen); *(name + USBUS_PREFIX_LEN + busnumlen) = '\0'; - err = pcap_add_if(alldevsp, name, PCAP_IF_UP, NULL, errbuf); - if (err != 0) { + /* + * There's an entry in this directory for every USB device, + * not for every bus; if there's more than one device on + * the bus, there'll be more than one entry for that bus, + * so we need to avoid adding multiple capture devices + * for each bus. + */ + if (find_or_add_dev(devlistp, name, PCAP_IF_UP, + get_usb_if_flags, NULL, errbuf) == NULL) { free(name); closedir(usbdir); - return (err); + return (PCAP_ERROR); } } free(name); @@ -2675,17 +2729,116 @@ finddevs_usb(pcap_if_t **alldevsp, char *errbuf) } #endif +/* + * Get additional flags for a device, using SIOCGIFMEDIA. + */ +#ifdef SIOCGIFMEDIA +static int +get_if_flags(const char *name, bpf_u_int32 *flags, char *errbuf) +{ + int sock; + struct ifmediareq req; + + sock = socket(AF_INET, SOCK_DGRAM, 0); + if (sock == -1) { + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, errno, + "Can't create socket to get media information for %s", + name); + return (-1); + } + memset(&req, 0, sizeof(req)); + strncpy(req.ifm_name, name, sizeof(req.ifm_name)); + if (ioctl(sock, SIOCGIFMEDIA, &req) < 0) { + if (errno == EOPNOTSUPP || errno == EINVAL || errno == ENOTTY || + errno == ENODEV) { + /* + * Not supported, so we can't provide any + * additional information. Assume that + * this means that "connected" vs. + * "disconnected" doesn't apply. + */ + *flags |= PCAP_IF_CONNECTION_STATUS_NOT_APPLICABLE; + close(sock); + return (0); + } + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, errno, + "SIOCGIFMEDIA on %s failed", name); + close(sock); + return (-1); + } + close(sock); + + /* + * OK, what type of network is this? + */ + switch (IFM_TYPE(req.ifm_active)) { + + case IFM_IEEE80211: + /* + * Wireless. + */ + *flags |= PCAP_IF_WIRELESS; + break; + } + + /* + * Do we know whether it's connected? + */ + if (req.ifm_status & IFM_AVALID) { + /* + * Yes. + */ + if (req.ifm_status & IFM_ACTIVE) { + /* + * It's connected. + */ + *flags |= PCAP_IF_CONNECTION_STATUS_CONNECTED; + } else { + /* + * It's disconnected. + */ + *flags |= PCAP_IF_CONNECTION_STATUS_DISCONNECTED; + } + } + return (0); +} +#else +static int +get_if_flags(const char *name _U_, bpf_u_int32 *flags _U_, char *errbuf _U_) +{ + /* + * Nothing we can do other than mark loopback devices as "the + * connected/disconnected status doesn't apply". + * + * XXX - on Solaris, can we do what the dladm command does, + * i.e. get a connected/disconnected indication from a kstat? + * (Note that you can also get the link speed, and possibly + * other information, from a kstat as well.) + */ + if (*flags & PCAP_IF_LOOPBACK) { + /* + * Loopback devices aren't wireless, and "connected"/ + * "disconnected" doesn't apply to them. + */ + *flags |= PCAP_IF_CONNECTION_STATUS_NOT_APPLICABLE; + return (0); + } + return (0); +} +#endif + int -pcap_platform_finddevs(pcap_if_t **alldevsp, char *errbuf) +pcap_platform_finddevs(pcap_if_list_t *devlistp, char *errbuf) { /* * Get the list of regular interfaces first. */ - if (pcap_findalldevs_interfaces(alldevsp, errbuf, check_bpf_bindable) == -1) + if (pcap_findalldevs_interfaces(devlistp, errbuf, check_bpf_bindable, + get_if_flags) == -1) return (-1); /* failure */ #if defined(__FreeBSD__) && defined(SIOCIFCREATE2) - if (finddevs_usb(alldevsp, errbuf) == -1) + if (finddevs_usb(devlistp, errbuf) == -1) return (-1); #endif @@ -2706,8 +2859,8 @@ monitor_mode(pcap_t *p, int set) sock = socket(AF_INET, SOCK_DGRAM, 0); if (sock == -1) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "can't open socket: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "can't open socket"); return (PCAP_ERROR); } @@ -2738,8 +2891,8 @@ monitor_mode(pcap_t *p, int set) return (PCAP_ERROR_RFMON_NOTSUP); default: - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "SIOCGIFMEDIA 1: %s", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "SIOCGIFMEDIA 1"); close(sock); return (PCAP_ERROR); } @@ -2758,15 +2911,15 @@ monitor_mode(pcap_t *p, int set) */ media_list = malloc(req.ifm_count * sizeof(*media_list)); if (media_list == NULL) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "malloc: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); close(sock); return (PCAP_ERROR); } req.ifm_ulist = media_list; if (ioctl(sock, SIOCGIFMEDIA, &req) < 0) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "SIOCGIFMEDIA: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "SIOCGIFMEDIA"); free(media_list); close(sock); return (PCAP_ERROR); @@ -2826,8 +2979,8 @@ monitor_mode(pcap_t *p, int set) sizeof(ifr.ifr_name)); ifr.ifr_media = req.ifm_current | IFM_IEEE80211_MONITOR; if (ioctl(sock, SIOCSIFMEDIA, &ifr) == -1) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "SIOCSIFMEDIA: %s", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, + PCAP_ERRBUF_SIZE, errno, "SIOCSIFMEDIA"); close(sock); return (PCAP_ERROR); } @@ -3051,8 +3204,8 @@ pcap_setfilter_bpf(pcap_t *p, struct bpf_program *fp) * some kernels. */ if (errno != EINVAL) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "BIOCSETF: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "BIOCSETF"); return (-1); } @@ -3080,11 +3233,10 @@ pcap_setdirection_bpf(pcap_t *p, pcap_direction_t d) direction = (d == PCAP_D_IN) ? BPF_D_IN : ((d == PCAP_D_OUT) ? BPF_D_OUT : BPF_D_INOUT); if (ioctl(p->fd, BIOCSDIRECTION, &direction) == -1) { - (void) pcap_snprintf(p->errbuf, sizeof(p->errbuf), - "Cannot set direction to %s: %s", + pcap_fmt_errmsg_for_errno(p->errbuf, sizeof(p->errbuf), + errno, "Cannot set direction to %s", (d == PCAP_D_IN) ? "PCAP_D_IN" : - ((d == PCAP_D_OUT) ? "PCAP_D_OUT" : "PCAP_D_INOUT"), - strerror(errno)); + ((d == PCAP_D_OUT) ? "PCAP_D_OUT" : "PCAP_D_INOUT")); return (-1); } return (0); @@ -3102,10 +3254,9 @@ pcap_setdirection_bpf(pcap_t *p, pcap_direction_t d) seesent = (d == PCAP_D_INOUT); if (ioctl(p->fd, BIOCSSEESENT, &seesent) == -1) { - (void) pcap_snprintf(p->errbuf, sizeof(p->errbuf), - "Cannot set direction to %s: %s", - (d == PCAP_D_INOUT) ? "PCAP_D_INOUT" : "PCAP_D_IN", - strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, sizeof(p->errbuf), + errno, "Cannot set direction to %s", + (d == PCAP_D_INOUT) ? "PCAP_D_INOUT" : "PCAP_D_IN"); return (-1); } return (0); @@ -3121,10 +3272,23 @@ pcap_set_datalink_bpf(pcap_t *p, int dlt) { #ifdef BIOCSDLT if (ioctl(p->fd, BIOCSDLT, &dlt) == -1) { - (void) pcap_snprintf(p->errbuf, sizeof(p->errbuf), - "Cannot set DLT %d: %s", dlt, strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, sizeof(p->errbuf), + errno, "Cannot set DLT %d", dlt); return (-1); } #endif return (0); } + +/* + * Platform-specific information. + */ +const char * +pcap_lib_version(void) +{ +#ifdef HAVE_ZEROCOPY_BPF + return (PCAP_VERSION_STRING " (with zerocopy support)"); +#else + return (PCAP_VERSION_STRING); +#endif +} diff --git a/freebsd/contrib/libpcap/pcap-common.c b/freebsd/contrib/libpcap/pcap-common.c index 27fcdcda..8219a031 100644 --- a/freebsd/contrib/libpcap/pcap-common.c +++ b/freebsd/contrib/libpcap/pcap-common.c @@ -20,26 +20,14 @@ * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. * - * pcap-common.c - common code for pcap and pcap-ng files + * pcap-common.c - common code for pcap and pcapng files */ #ifdef HAVE_CONFIG_H -#include "config.h" +#include <config.h> #endif -#ifdef _WIN32 -#include <pcap-stdinc.h> -#else /* _WIN32 */ -#if HAVE_INTTYPES_H -#include <inttypes.h> -#elif HAVE_STDINT_H -#include <stdint.h> -#endif -#ifdef HAVE_SYS_BITYPES_H -#include <sys/bitypes.h> -#endif -#include <sys/types.h> -#endif /* _WIN32 */ +#include <pcap-types.h> #include "pcap-int.h" #include "extract.h" @@ -498,9 +486,14 @@ /* * IEEE 802.15.4, exactly as it appears in the spec (no padding, no - * nothing); requested by Mikko Saarnivala <mikko.saarnivala@sensinode.com>. + * nothing), and with the FCS at the end of the frame; requested by + * Mikko Saarnivala <mikko.saarnivala@sensinode.com>. + * + * This should only be used if the FCS is present at the end of the + * frame; if the frame has no FCS, DLT_IEEE802_15_4_NOFCS should be + * used. */ -#define LINKTYPE_IEEE802_15_4 195 +#define LINKTYPE_IEEE802_15_4_WITHFCS 195 /* * Various link-layer types, with a pseudo-header, for SITA @@ -705,14 +698,14 @@ * the pseudo-header is: * * struct dl_ipnetinfo { - * u_int8_t dli_version; - * u_int8_t dli_family; - * u_int16_t dli_htype; - * u_int32_t dli_pktlen; - * u_int32_t dli_ifindex; - * u_int32_t dli_grifindex; - * u_int32_t dli_zsrc; - * u_int32_t dli_zdst; + * uint8_t dli_version; + * uint8_t dli_family; + * uint16_t dli_htype; + * uint32_t dli_pktlen; + * uint32_t dli_ifindex; + * uint32_t dli_grifindex; + * uint32_t dli_zsrc; + * uint32_t dli_zdst; * }; * * dli_version is 2 for the current version of the pseudo-header. @@ -889,7 +882,7 @@ /* * pfsync output; DLT_PFSYNC is 18, which collides with DLT_CIP in - * SuSE 6.3, on OpenBSD, NetBSD, DragonFly BSD, and Mac OS X, and + * SuSE 6.3, on OpenBSD, NetBSD, DragonFly BSD, and macOS, and * is 121, which collides with DLT_HHDLC, in FreeBSD. We pick a * shiny new link-layer header type value that doesn't collide with * anything, in the hopes that future pfsync savefiles, if any, @@ -986,7 +979,7 @@ * So I'll just give them one; hopefully this will show up in a * libpcap release in time for them to get this into 10.10 Big Sur * or whatever Mavericks' successor is called. LINKTYPE_PKTAP - * will be 258 *even on OS X*; that is *intentional*, so that + * will be 258 *even on macOS*; that is *intentional*, so that * PKTAP files look the same on *all* OSes (different OSes can have * different numerical values for a given DLT_, but *MUST NOT* have * different values for what goes in a file, as files can be moved @@ -1030,7 +1023,67 @@ */ #define LINKTYPE_RDS 265 -#define LINKTYPE_MATCHING_MAX 265 /* highest value in the "matching" range */ +/* + * USB packets, beginning with a Darwin (macOS, etc.) header. + */ +#define LINKTYPE_USB_DARWIN 266 + +/* + * OpenBSD DLT_OPENFLOW. + */ +#define LINKTYPE_OPENFLOW 267 + +/* + * SDLC frames containing SNA PDUs. + */ +#define LINKTYPE_SDLC 268 + +/* + * per "Selvig, Bjorn" <b.selvig@ti.com> used for + * TI protocol sniffer. + */ +#define LINKTYPE_TI_LLN_SNIFFER 269 + +/* + * per: Erik de Jong <erikdejong at gmail.com> for + * https://github.com/eriknl/LoRaTap/releases/tag/v0.1 + */ +#define LINKTYPE_LORATAP 270 + +/* + * per: Stefanha at gmail.com for + * http://lists.sandelman.ca/pipermail/tcpdump-workers/2017-May/000772.html + * and: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/uapi/linux/vsockmon.h + * for: http://qemu-project.org/Features/VirtioVsock + */ +#define LINKTYPE_VSOCK 271 + +/* + * Nordic Semiconductor Bluetooth LE sniffer. + */ +#define LINKTYPE_NORDIC_BLE 272 + +/* + * Excentis DOCSIS 3.1 RF sniffer (XRA-31) + * per: bruno.verstuyft at excentis.com + * http://www.xra31.com/xra-header + */ +#define LINKTYPE_DOCSIS31_XRA31 273 + +/* + * mPackets, as specified by IEEE 802.3br Figure 99-4, starting + * with the preamble and always ending with a CRC field. + */ +#define LINKTYPE_ETHERNET_MPACKET 274 + +/* + * DisplayPort AUX channel monitoring data as specified by VESA + * DisplayPort(DP) Standard preceeded by a pseudo-header. + * per dirk.eibach at gdsys.cc + */ +#define LINKTYPE_DISPLAYPORT_AUX 275 + +#define LINKTYPE_MATCHING_MAX 275 /* highest value in the "matching" range */ static struct linktype_map { int dlt; @@ -1177,7 +1230,22 @@ linktype_to_dlt(int linktype) return linktype; } -#define EXTRACT_ +/* + * Return the maximum snapshot length for a given DLT_ value. + * + * For most link-layer types, we use MAXIMUM_SNAPLEN, but for DLT_DBUS, + * the maximum is 134217728, as per + * + * https://dbus.freedesktop.org/doc/dbus-specification.html#message-protocol-messages + */ +u_int +max_snaplen_for_dlt(int dlt) +{ + if (dlt == DLT_DBUS) + return 134217728; + else + return MAXIMUM_SNAPLEN; +} /* * DLT_LINUX_SLL packets with a protocol type of LINUX_SLL_P_CAN or @@ -1194,7 +1262,7 @@ swap_linux_sll_header(const struct pcap_pkthdr *hdr, u_char *buf) u_int caplen = hdr->caplen; u_int length = hdr->len; struct sll_header *shdr = (struct sll_header *)buf; - u_int16_t protocol; + uint16_t protocol; pcap_can_socketcan_hdr *chdr; if (caplen < (u_int) sizeof(struct sll_header) || @@ -1340,7 +1408,7 @@ swap_linux_usb_header(const struct pcap_pkthdr *hdr, u_char *buf, if (uhdr->transfer_type == URB_ISOCHRONOUS) { /* swap the values in struct linux_usb_isodesc */ usb_isodesc *pisodesc; - u_int32_t i; + uint32_t i; pisodesc = (usb_isodesc *)(void *)(buf+offset); for (i = 0; i < uhdr->ndesc; i++) { @@ -1386,7 +1454,7 @@ swap_nflog_header(const struct pcap_pkthdr *hdr, u_char *buf) nflog_tlv_t *tlv; u_int caplen = hdr->caplen; u_int length = hdr->len; - u_int16_t size; + uint16_t size; if (caplen < (u_int) sizeof(nflog_hdr_t) || length < (u_int) sizeof(nflog_hdr_t)) { diff --git a/freebsd/contrib/libpcap/pcap-common.h b/freebsd/contrib/libpcap/pcap-common.h index 6ac5bcd2..88c057cb 100644 --- a/freebsd/contrib/libpcap/pcap-common.h +++ b/freebsd/contrib/libpcap/pcap-common.h @@ -1,3 +1,25 @@ +/* + * Copyright (c) 1993, 1994, 1995, 1996, 1997 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that: (1) source code distributions + * retain the above copyright notice and this paragraph in its entirety, (2) + * distributions including binary code include the above copyright notice and + * this paragraph in its entirety in the documentation or other materials + * provided with the distribution, and (3) all advertising materials mentioning + * features or use of this software display the following acknowledgement: + * ``This product includes software developed by the University of California, + * Lawrence Berkeley Laboratory and its contributors.'' Neither the name of + * the University nor the names of its contributors may be used to endorse + * or promote products derived from this software without specific prior + * written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + * + * pcap-common.h - common code for pcap and pcapng files + */ /* * We use the "receiver-makes-right" approach to byte order, @@ -23,3 +45,5 @@ extern int linktype_to_dlt(int linktype); extern void swap_pseudo_headers(int linktype, struct pcap_pkthdr *hdr, u_char *data); + +extern u_int max_snaplen_for_dlt(int dlt); diff --git a/freebsd/contrib/libpcap/pcap-int.h b/freebsd/contrib/libpcap/pcap-int.h index 7db7ff5e..5888df72 100644 --- a/freebsd/contrib/libpcap/pcap-int.h +++ b/freebsd/contrib/libpcap/pcap-int.h @@ -34,55 +34,40 @@ #ifndef pcap_int_h #define pcap_int_h +#include <signal.h> + #include <pcap/pcap.h> +#include "varattrs.h" +#include "fmtutils.h" + +/* + * Version string. + * Uses PACKAGE_VERSION from config.h. + */ +#define PCAP_VERSION_STRING "libpcap version " PACKAGE_VERSION + #ifdef __cplusplus extern "C" { #endif -#if defined(_WIN32) - /* - * Make sure Packet32.h doesn't define BPF structures that we've - * probably already defined as a result of including <pcap/pcap.h>. - */ - #define BPF_MAJOR_VERSION - #include <Packet32.h> -#elif defined(MSDOS) +#ifdef MSDOS #include <fcntl.h> #include <io.h> #endif -#if (defined(_MSC_VER) && (_MSC_VER <= 1200)) /* we are compiling with Visual Studio 6, that doesn't support the LL suffix*/ - -/* - * Swap byte ordering of unsigned long long timestamp on a big endian - * machine. - */ -#define SWAPLL(ull) ((ull & 0xff00000000000000) >> 56) | \ - ((ull & 0x00ff000000000000) >> 40) | \ - ((ull & 0x0000ff0000000000) >> 24) | \ - ((ull & 0x000000ff00000000) >> 8) | \ - ((ull & 0x00000000ff000000) << 8) | \ - ((ull & 0x0000000000ff0000) << 24) | \ - ((ull & 0x000000000000ff00) << 40) | \ - ((ull & 0x00000000000000ff) << 56) - -#else /* A recent Visual studio compiler or not VC */ - /* * Swap byte ordering of unsigned long long timestamp on a big endian * machine. */ -#define SWAPLL(ull) ((ull & 0xff00000000000000LL) >> 56) | \ - ((ull & 0x00ff000000000000LL) >> 40) | \ - ((ull & 0x0000ff0000000000LL) >> 24) | \ - ((ull & 0x000000ff00000000LL) >> 8) | \ - ((ull & 0x00000000ff000000LL) << 8) | \ - ((ull & 0x0000000000ff0000LL) << 24) | \ - ((ull & 0x000000000000ff00LL) << 40) | \ - ((ull & 0x00000000000000ffLL) << 56) - -#endif /* _MSC_VER */ +#define SWAPLL(ull) ((ull & 0xff00000000000000ULL) >> 56) | \ + ((ull & 0x00ff000000000000ULL) >> 40) | \ + ((ull & 0x0000ff0000000000ULL) >> 24) | \ + ((ull & 0x000000ff00000000ULL) >> 8) | \ + ((ull & 0x00000000ff000000ULL) << 8) | \ + ((ull & 0x0000000000ff0000ULL) << 24) | \ + ((ull & 0x000000000000ff00ULL) << 40) | \ + ((ull & 0x00000000000000ffULL) << 56) /* * Maximum snapshot length. @@ -114,19 +99,32 @@ struct pcap_opt { int promisc; int rfmon; /* monitor mode */ int immediate; /* immediate mode - deliver packets as soon as they arrive */ + int nonblock; /* non-blocking mode - don't wait for packets to be delivered, return "no packets available" */ int tstamp_type; int tstamp_precision; + + /* + * Platform-dependent options. + */ +#ifdef __linux__ + int protocol; /* protocol to use when creating PF_PACKET socket */ +#endif +#ifdef _WIN32 + int nocapture_local;/* disable NPF loopback */ +#endif }; typedef int (*activate_op_t)(pcap_t *); typedef int (*can_set_rfmon_op_t)(pcap_t *); typedef int (*read_op_t)(pcap_t *, int cnt, pcap_handler, u_char *); +typedef int (*next_packet_op_t)(pcap_t *, struct pcap_pkthdr *, u_char **); typedef int (*inject_op_t)(pcap_t *, const void *, size_t); +typedef void (*save_current_filter_op_t)(pcap_t *, const char *); typedef int (*setfilter_op_t)(pcap_t *, struct bpf_program *); typedef int (*setdirection_op_t)(pcap_t *, pcap_direction_t); typedef int (*set_datalink_op_t)(pcap_t *, int); -typedef int (*getnonblock_op_t)(pcap_t *, char *); -typedef int (*setnonblock_op_t)(pcap_t *, int, char *); +typedef int (*getnonblock_op_t)(pcap_t *); +typedef int (*setnonblock_op_t)(pcap_t *, int); typedef int (*stats_op_t)(pcap_t *, struct pcap_stat *); #ifdef _WIN32 typedef struct pcap_stat *(*stats_ex_op_t)(pcap_t *, int *); @@ -155,15 +153,14 @@ struct pcap { read_op_t read_op; /* - * Method to call to read packets from a savefile. + * Method to call to read the next packet from a savefile. */ - int (*next_packet_op)(pcap_t *, struct pcap_pkthdr *, u_char **); + next_packet_op_t next_packet_op; #ifdef _WIN32 - ADAPTER *adapter; + HANDLE handle; #else int fd; - int selectable_fd; #endif /* _WIN32 */ /* @@ -174,10 +171,14 @@ struct pcap { u_char *bp; int cc; - int break_loop; /* flag set to force break from packet-reading loop */ + sig_atomic_t break_loop; /* flag set to force break from packet-reading loop */ void *priv; /* private data for methods */ +#ifdef ENABLE_REMOTE + struct pcap_samp rmt_samp; /* parameters related to the sampling process. */ +#endif + int swapped; FILE *rfile; /* null if live capture, non-null if savefile */ u_int fddipad; @@ -208,7 +209,7 @@ struct pcap { u_char *pkt; #ifdef _WIN32 - struct pcap_stat stat; /* used for pcap_stats_ex() */ + struct pcap_stat stat; /* used for pcap_stats_ex() */ #endif /* We're accepting only packets in this direction/these directions. */ @@ -219,6 +220,23 @@ struct pcap { */ int bpf_codegen_flags; +#if !defined(_WIN32) && !defined(MSDOS) + int selectable_fd; /* FD on which select()/poll()/epoll_wait()/kevent()/etc. can be done */ + + /* + * In case there either is no selectable FD, or there is but + * it doesn't necessarily work (e.g., if it doesn't get notified + * if the packet capture timeout expires before the buffer + * fills up), this points to a timeout that should be used + * in select()/poll()/epoll_wait()/kevent() call. The pcap_t should + * be put into non-blocking mode, and, if the timeout expires on + * the call, an attempt should be made to read packets from all + * pcap_t's with a required timeout, and the code must be + * prepared not to see any packets from the attempt. + */ + struct timeval *required_select_timeout; +#endif + /* * Placeholder for filter code if bpf not in kernel. */ @@ -240,6 +258,7 @@ struct pcap { activate_op_t activate_op; can_set_rfmon_op_t can_set_rfmon_op; inject_op_t inject_op; + save_current_filter_op_t save_current_filter_op; setfilter_op_t setfilter_op; setdirection_op_t setdirection_op; set_datalink_op_t set_datalink_op; @@ -378,8 +397,8 @@ int pcap_offline_read(pcap_t *, int, pcap_handler, u_char *); * Routines that most pcap implementations can use for non-blocking mode. */ #if !defined(_WIN32) && !defined(MSDOS) -int pcap_getnonblock_fd(pcap_t *, char *); -int pcap_setnonblock_fd(pcap_t *p, int, char *); +int pcap_getnonblock_fd(pcap_t *); +int pcap_setnonblock_fd(pcap_t *p, int); #endif /* @@ -404,32 +423,47 @@ int pcap_check_activated(pcap_t *); /* * Internal interfaces for "pcap_findalldevs()". * - * "pcap_platform_finddevs()" is a platform-dependent routine to + * A pcap_if_list_t * is a reference to a list of devices. + * + * A get_if_flags_func is a platform-dependent function called to get + * additional interface flags. + * + * "pcap_platform_finddevs()" is the platform-dependent routine to * find local network interfaces. * * "pcap_findalldevs_interfaces()" is a helper to find those interfaces * using the "standard" mechanisms (SIOCGIFCONF, "getifaddrs()", etc.). * - * "pcap_add_if()" adds an interface to the list of interfaces, for - * use by various "find interfaces" routines. + * "add_dev()" adds an entry to a pcap_if_list_t. + * + * "find_dev()" tries to find a device, by name, in a pcap_if_list_t. + * + * "find_or_add_dev()" checks whether a device is already in a pcap_if_list_t + * and, if not, adds an entry for it. */ -int pcap_platform_finddevs(pcap_if_t **, char *); +struct pcap_if_list; +typedef struct pcap_if_list pcap_if_list_t; +typedef int (*get_if_flags_func)(const char *, bpf_u_int32 *, char *); +int pcap_platform_finddevs(pcap_if_list_t *, char *); #if !defined(_WIN32) && !defined(MSDOS) -int pcap_findalldevs_interfaces(pcap_if_t **, char *, - int (*)(const char *)); +int pcap_findalldevs_interfaces(pcap_if_list_t *, char *, + int (*)(const char *), get_if_flags_func); #endif -int add_addr_to_iflist(pcap_if_t **, const char *, bpf_u_int32, - struct sockaddr *, size_t, struct sockaddr *, size_t, - struct sockaddr *, size_t, struct sockaddr *, size_t, char *); +pcap_if_t *find_or_add_dev(pcap_if_list_t *, const char *, bpf_u_int32, + get_if_flags_func, const char *, char *); +pcap_if_t *find_dev(pcap_if_list_t *, const char *); +pcap_if_t *add_dev(pcap_if_list_t *, const char *, bpf_u_int32, const char *, + char *); int add_addr_to_dev(pcap_if_t *, struct sockaddr *, size_t, struct sockaddr *, size_t, struct sockaddr *, size_t, struct sockaddr *dstaddr, size_t, char *errbuf); -int pcap_add_if(pcap_if_t **, const char *, bpf_u_int32, const char *, - char *); -int add_or_find_if(pcap_if_t **, pcap_if_t **, const char *, bpf_u_int32, - const char *, char *); #ifndef _WIN32 -bpf_u_int32 if_flags_to_pcap_flags(const char *, u_int); +pcap_if_t *find_or_add_if(pcap_if_list_t *, const char *, bpf_u_int32, + get_if_flags_func, char *); +int add_addr_to_if(pcap_if_list_t *, const char *, bpf_u_int32, + get_if_flags_func, + struct sockaddr *, size_t, struct sockaddr *, size_t, + struct sockaddr *, size_t, struct sockaddr *, size_t, char *); #endif /* @@ -462,6 +496,10 @@ int install_bpf_program(pcap_t *, struct bpf_program *); int pcap_strcasecmp(const char *, const char *); +#ifdef YYDEBUG +extern int pcap_debug; +#endif + #ifdef __cplusplus } #endif diff --git a/freebsd/contrib/libpcap/pcap-types.h b/freebsd/contrib/libpcap/pcap-types.h new file mode 100644 index 00000000..9614f9f1 --- /dev/null +++ b/freebsd/contrib/libpcap/pcap-types.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2002 - 2005 NetGroup, Politecnico di Torino (Italy) + * Copyright (c) 2005 - 2009 CACE Technologies, Inc. Davis (California) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the Politecnico di Torino nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef pcap_types_h +#define pcap_types_h + +/* + * Get u_int defined, by hook or by crook. + */ +#ifdef _WIN32 + + /* + * This defines u_int. + */ + #include <winsock2.h> +#else /* _WIN32 */ + /* + * This defines u_int, among other types. + */ + #include <sys/types.h> +#endif + +#endif /* pcap_types_h */ diff --git a/freebsd/contrib/libpcap/pcap.c b/freebsd/contrib/libpcap/pcap.c index c7e7d02e..4cf53b22 100644 --- a/freebsd/contrib/libpcap/pcap.c +++ b/freebsd/contrib/libpcap/pcap.c @@ -34,24 +34,28 @@ */ #ifdef HAVE_CONFIG_H -#include "config.h" +#include <config.h> #endif -#ifdef _WIN32 -#include <pcap-stdinc.h> -#else /* _WIN32 */ -#if HAVE_INTTYPES_H -#include <inttypes.h> -#elif HAVE_STDINT_H -#include <stdint.h> +#include <pcap-types.h> +#ifndef _WIN32 +#include <sys/param.h> +#ifndef MSDOS +#include <sys/file.h> #endif -#ifdef HAVE_SYS_BITYPES_H -#include <sys/bitypes.h> +#include <sys/ioctl.h> +#include <sys/socket.h> +#ifdef HAVE_SYS_SOCKIO_H +#include <sys/sockio.h> #endif -#include <sys/types.h> -#include <sys/mman.h> + +struct mbuf; /* Squelch compiler warnings on some platforms for */ +struct rtentry; /* declarations in <net/if.h> */ +#include <net/if.h> +#include <netinet/in.h> #endif /* _WIN32 */ +#include <ctype.h> #include <stdio.h> #include <stdlib.h> #include <string.h> @@ -60,6 +64,11 @@ #endif #include <fcntl.h> #include <errno.h> +#ifdef HAVE_LIMITS_H +#include <limits.h> +#else +#define INT_MAX 2147483647 +#endif #ifdef HAVE_OS_PROTO_H #include "os-proto.h" @@ -71,6 +80,8 @@ #include "pcap-int.h" +#include "optimize.h" + #ifdef HAVE_DAG_API #include "pcap-dag.h" #endif /* HAVE_DAG_API */ @@ -104,16 +115,88 @@ #endif #ifdef PCAP_SUPPORT_NETMAP -pcap_t* pcap_netmap_create(const char *device, char *ebuf, int *is_ours); +#include "pcap-netmap.h" #endif #ifdef PCAP_SUPPORT_DBUS #include "pcap-dbus.h" #endif +#ifdef PCAP_SUPPORT_RDMASNIFF +#include "pcap-rdmasniff.h" +#endif + +#ifdef _WIN32 +/* + * DllMain(), required when built as a Windows DLL. + */ +BOOL WINAPI DllMain( + HANDLE hinstDLL, + DWORD dwReason, + LPVOID lpvReserved +) +{ + return (TRUE); +} + +/* + * Start WinSock. + * Exported in case some applications using WinPcap called it, + * even though it wasn't exported. + */ +int +wsockinit(void) +{ + WORD wVersionRequested; + WSADATA wsaData; + static int err = -1; + static int done = 0; + + if (done) + return (err); + + wVersionRequested = MAKEWORD( 1, 1); + err = WSAStartup( wVersionRequested, &wsaData ); + atexit ((void(*)(void))WSACleanup); + done = 1; + + if ( err != 0 ) + err = -1; + return (err); +} + +/* + * This is the exported function; new programs should call this. + */ +int +pcap_wsockinit(void) +{ + return (wsockinit()); +} +#endif /* _WIN32 */ + +/* + * String containing the library version. + * Not explicitly exported via a header file - the right API to use + * is pcap_lib_version() - but some programs included it, so we + * provide it. + * + * We declare it here, right before defining it, to squelch any + * warnings we might get from compilers about the lack of a + * declaration. + */ +PCAP_API char pcap_version[]; +PCAP_API_DEF char pcap_version[] = PACKAGE_VERSION; + static int pcap_not_initialized(pcap_t *pcap) { + if (pcap->activated) { + /* A module probably forgot to set the function pointer */ + (void)pcap_snprintf(pcap->errbuf, sizeof(pcap->errbuf), + "This operation isn't properly handled by that device"); + return (PCAP_ERROR); + } /* in case the caller doesn't check for PCAP_ERROR_NOT_ACTIVATED */ (void)pcap_snprintf(pcap->errbuf, sizeof(pcap->errbuf), "This handle hasn't been activated yet"); @@ -125,6 +208,12 @@ pcap_not_initialized(pcap_t *pcap) static void * pcap_not_initialized_ptr(pcap_t *pcap) { + if (pcap->activated) { + /* A module probably forgot to set the function pointer */ + (void)pcap_snprintf(pcap->errbuf, sizeof(pcap->errbuf), + "This operation isn't properly handled by that device"); + return (NULL); + } (void)pcap_snprintf(pcap->errbuf, sizeof(pcap->errbuf), "This handle hasn't been activated yet"); return (NULL); @@ -133,6 +222,12 @@ pcap_not_initialized_ptr(pcap_t *pcap) static HANDLE pcap_getevent_not_initialized(pcap_t *pcap) { + if (pcap->activated) { + /* A module probably forgot to set the function pointer */ + (void)pcap_snprintf(pcap->errbuf, sizeof(pcap->errbuf), + "This operation isn't properly handled by that device"); + return (INVALID_HANDLE_VALUE); + } (void)pcap_snprintf(pcap->errbuf, sizeof(pcap->errbuf), "This handle hasn't been activated yet"); return (INVALID_HANDLE_VALUE); @@ -141,6 +236,12 @@ pcap_getevent_not_initialized(pcap_t *pcap) static u_int pcap_sendqueue_transmit_not_initialized(pcap_t *pcap, pcap_send_queue* queue, int sync) { + if (pcap->activated) { + /* A module probably forgot to set the function pointer */ + (void)pcap_snprintf(pcap->errbuf, sizeof(pcap->errbuf), + "This operation isn't properly handled by that device"); + return (0); + } (void)pcap_snprintf(pcap->errbuf, sizeof(pcap->errbuf), "This handle hasn't been activated yet"); return (0); @@ -149,6 +250,12 @@ pcap_sendqueue_transmit_not_initialized(pcap_t *pcap, pcap_send_queue* queue, in static PAirpcapHandle pcap_get_airpcap_handle_not_initialized(pcap_t *pcap) { + if (pcap->activated) { + /* A module probably forgot to set the function pointer */ + (void)pcap_snprintf(pcap->errbuf, sizeof(pcap->errbuf), + "This operation isn't properly handled by that device"); + return (NULL); + } (void)pcap_snprintf(pcap->errbuf, sizeof(pcap->errbuf), "This handle hasn't been activated yet"); return (NULL); @@ -197,8 +304,8 @@ pcap_list_tstamp_types(pcap_t *p, int **tstamp_typesp) *tstamp_typesp = (int*)calloc(sizeof(**tstamp_typesp), p->tstamp_type_count); if (*tstamp_typesp == NULL) { - (void)pcap_snprintf(p->errbuf, sizeof(p->errbuf), - "malloc: %s", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, sizeof(p->errbuf), + errno, "malloc"); return (PCAP_ERROR); } (void)memcpy(*tstamp_typesp, p->tstamp_type_list, @@ -302,13 +409,17 @@ pcap_next_ex(pcap_t *p, struct pcap_pkthdr **pkt_header, return (p->read_op(p, 1, p->oneshot_callback, (u_char *)&s)); } +/* + * Implementation of a pcap_if_list_t. + */ +struct pcap_if_list { + pcap_if_t *beginning; +}; + static struct capture_source_type { - int (*findalldevs_op)(pcap_if_t **, char *); + int (*findalldevs_op)(pcap_if_list_t *, char *); pcap_t *(*create_op)(const char *, char *, int *); } capture_source_types[] = { -#ifdef PCAP_SUPPORT_NETMAP - { NULL, pcap_netmap_create }, -#endif #ifdef HAVE_DAG_API { dag_findalldevs, dag_create }, #endif @@ -333,9 +444,15 @@ static struct capture_source_type { #ifdef PCAP_SUPPORT_NETFILTER { netfilter_findalldevs, netfilter_create }, #endif +#ifdef PCAP_SUPPORT_NETMAP + { pcap_netmap_findalldevs, pcap_netmap_create }, +#endif #ifdef PCAP_SUPPORT_DBUS { dbus_findalldevs, dbus_create }, #endif +#ifdef PCAP_SUPPORT_RDMASNIFF + { rdmasniff_findalldevs, rdmasniff_create }, +#endif { NULL, NULL } }; @@ -349,35 +466,1503 @@ int pcap_findalldevs(pcap_if_t **alldevsp, char *errbuf) { size_t i; + pcap_if_list_t devlist; /* * Find all the local network interfaces on which we * can capture. */ - if (pcap_platform_finddevs(alldevsp, errbuf) == -1) + devlist.beginning = NULL; + if (pcap_platform_finddevs(&devlist, errbuf) == -1) { + /* + * Failed - free all of the entries we were given + * before we failed. + */ + if (devlist.beginning != NULL) + pcap_freealldevs(devlist.beginning); + *alldevsp = NULL; return (-1); + } /* * Ask each of the non-local-network-interface capture * source types what interfaces they have. */ for (i = 0; capture_source_types[i].findalldevs_op != NULL; i++) { - if (capture_source_types[i].findalldevs_op(alldevsp, errbuf) == -1) { + if (capture_source_types[i].findalldevs_op(&devlist, errbuf) == -1) { /* * We had an error; free the list we've been * constructing. */ - if (*alldevsp != NULL) { - pcap_freealldevs(*alldevsp); - *alldevsp = NULL; + if (devlist.beginning != NULL) + pcap_freealldevs(devlist.beginning); + *alldevsp = NULL; + return (-1); + } + } + + /* + * Return the first entry of the list of all devices. + */ + *alldevsp = devlist.beginning; + return (0); +} + +static struct sockaddr * +dup_sockaddr(struct sockaddr *sa, size_t sa_length) +{ + struct sockaddr *newsa; + + if ((newsa = malloc(sa_length)) == NULL) + return (NULL); + return (memcpy(newsa, sa, sa_length)); +} + +/* + * Construct a "figure of merit" for an interface, for use when sorting + * the list of interfaces, in which interfaces that are up are superior + * to interfaces that aren't up, interfaces that are up and running are + * superior to interfaces that are up but not running, and non-loopback + * interfaces that are up and running are superior to loopback interfaces, + * and interfaces with the same flags have a figure of merit that's higher + * the lower the instance number. + * + * The goal is to try to put the interfaces most likely to be useful for + * capture at the beginning of the list. + * + * The figure of merit, which is lower the "better" the interface is, + * has the uppermost bit set if the interface isn't running, the bit + * below that set if the interface isn't up, the bit below that set + * if the interface is a loopback interface, and the interface index + * in the 29 bits below that. (Yes, we assume u_int is 32 bits.) + */ +static u_int +get_figure_of_merit(pcap_if_t *dev) +{ + const char *cp; + u_int n; + + if (strcmp(dev->name, "any") == 0) { + /* + * Give the "any" device an artificially high instance + * number, so it shows up after all other non-loopback + * interfaces. + */ + n = 0x1FFFFFFF; /* 29 all-1 bits */ + } else { + /* + * A number at the end of the device name string is + * assumed to be an instance number. Add 1 to the + * instance number, and use 0 for "no instance + * number", so we don't put "no instance number" + * devices and "instance 0" devices together. + */ + cp = dev->name + strlen(dev->name) - 1; + while (cp-1 >= dev->name && *(cp-1) >= '0' && *(cp-1) <= '9') + cp--; + if (*cp >= '0' && *cp <= '9') + n = atoi(cp) + 1; + else + n = 0; + } + if (!(dev->flags & PCAP_IF_RUNNING)) + n |= 0x80000000; + if (!(dev->flags & PCAP_IF_UP)) + n |= 0x40000000; + + /* + * Give non-wireless interfaces that aren't disconnected a better + * figure of merit than interfaces that are disconnected, as + * "disconnected" should indicate that the interface isn't + * plugged into a network and thus won't give you any traffic. + * + * For wireless interfaces, it means "associated with a network", + * which we presume not to necessarily prevent capture, as you + * might run the adapter in some flavor of monitor mode. + */ + if (!(dev->flags & PCAP_IF_WIRELESS) && + (dev->flags & PCAP_IF_CONNECTION_STATUS) == PCAP_IF_CONNECTION_STATUS_DISCONNECTED) + n |= 0x20000000; + + /* + * Sort loopback devices after non-loopback devices, *except* for + * disconnected devices. + */ + if (dev->flags & PCAP_IF_LOOPBACK) + n |= 0x10000000; + + return (n); +} + +#ifndef _WIN32 +/* + * Try to get a description for a given device. + * Returns a mallocated description if it could and NULL if it couldn't. + * + * XXX - on FreeBSDs that support it, should it get the sysctl named + * "dev.{adapter family name}.{adapter unit}.%desc" to get a description + * of the adapter? Note that "dev.an.0.%desc" is "Aironet PC4500/PC4800" + * with my Cisco 350 card, so the name isn't entirely descriptive. The + * "dev.an.0.%pnpinfo" has a better description, although one might argue + * that the problem is really a driver bug - if it can find out that it's + * a Cisco 340 or 350, rather than an old Aironet card, it should use + * that in the description. + * + * Do NetBSD, DragonflyBSD, or OpenBSD support this as well? FreeBSD + * and OpenBSD let you get a description, but it's not generated by the OS, + * it's set with another ioctl that ifconfig supports; we use that to get + * a description in FreeBSD and OpenBSD, but if there is no such + * description available, it still might be nice to get some description + * string based on the device type or something such as that. + * + * In macOS, the System Configuration framework can apparently return + * names in 10.4 and later. + * + * It also appears that freedesktop.org's HAL offers an "info.product" + * string, but the HAL specification says it "should not be used in any + * UI" and "subsystem/capability specific properties" should be used + * instead and, in any case, I think HAL is being deprecated in + * favor of other stuff such as DeviceKit. DeviceKit doesn't appear + * to have any obvious product information for devices, but maybe + * I haven't looked hard enough. + * + * Using the System Configuration framework, or HAL, or DeviceKit, or + * whatever, would require that libpcap applications be linked with + * the frameworks/libraries in question. That shouldn't be a problem + * for programs linking with the shared version of libpcap (unless + * you're running on AIX - which I think is the only UN*X that doesn't + * support linking a shared library with other libraries on which it + * depends, and having an executable linked only with the first shared + * library automatically pick up the other libraries when started - + * and using HAL or whatever). Programs linked with the static + * version of libpcap would have to use pcap-config with the --static + * flag in order to get the right linker flags in order to pick up + * the additional libraries/frameworks; those programs need that anyway + * for libpcap 1.1 and beyond on Linux, as, by default, it requires + * -lnl. + * + * Do any other UN*Xes, or desktop environments support getting a + * description? + */ +static char * +#ifdef SIOCGIFDESCR +get_if_description(const char *name) +{ + char *description = NULL; + int s; + struct ifreq ifrdesc; +#ifndef IFDESCRSIZE + size_t descrlen = 64; +#else + size_t descrlen = IFDESCRSIZE; +#endif /* IFDESCRSIZE */ + + /* + * Get the description for the interface. + */ + memset(&ifrdesc, 0, sizeof ifrdesc); + strlcpy(ifrdesc.ifr_name, name, sizeof ifrdesc.ifr_name); + s = socket(AF_INET, SOCK_DGRAM, 0); + if (s >= 0) { +#ifdef __FreeBSD__ + /* + * On FreeBSD, if the buffer isn't big enough for the + * description, the ioctl succeeds, but the description + * isn't copied, ifr_buffer.length is set to the description + * length, and ifr_buffer.buffer is set to NULL. + */ + for (;;) { + free(description); + if ((description = malloc(descrlen)) != NULL) { + ifrdesc.ifr_buffer.buffer = description; + ifrdesc.ifr_buffer.length = descrlen; + if (ioctl(s, SIOCGIFDESCR, &ifrdesc) == 0) { + if (ifrdesc.ifr_buffer.buffer == + description) + break; + else + descrlen = ifrdesc.ifr_buffer.length; + } else { + /* + * Failed to get interface description. + */ + free(description); + description = NULL; + break; + } + } else + break; + } +#else /* __FreeBSD__ */ + /* + * The only other OS that currently supports + * SIOCGIFDESCR is OpenBSD, and it has no way + * to get the description length - it's clamped + * to a maximum of IFDESCRSIZE. + */ + if ((description = malloc(descrlen)) != NULL) { + ifrdesc.ifr_data = (caddr_t)description; + if (ioctl(s, SIOCGIFDESCR, &ifrdesc) != 0) { + /* + * Failed to get interface description. + */ + free(description); + description = NULL; } + } +#endif /* __FreeBSD__ */ + close(s); + if (description != NULL && strlen(description) == 0) { + /* + * Description is empty, so discard it. + */ + free(description); + description = NULL; + } + } + +#ifdef __FreeBSD__ + /* + * For FreeBSD, if we didn't get a description, and this is + * a device with a name of the form usbusN, label it as a USB + * bus. + */ + if (description == NULL) { + if (strncmp(name, "usbus", 5) == 0) { + /* + * OK, it begins with "usbus". + */ + long busnum; + char *p; + + errno = 0; + busnum = strtol(name + 5, &p, 10); + if (errno == 0 && p != name + 5 && *p == '\0' && + busnum >= 0 && busnum <= INT_MAX) { + /* + * OK, it's a valid number that's not + * bigger than INT_MAX. Construct + * a description from it. + */ + static const char descr_prefix[] = "USB bus number "; + size_t descr_size; + + /* + * Allow enough room for a 32-bit bus number. + * sizeof (descr_prefix) includes the + * terminating NUL. + */ + descr_size = sizeof (descr_prefix) + 10; + description = malloc(descr_size); + if (description != NULL) { + pcap_snprintf(description, descr_size, + "%s%ld", descr_prefix, busnum); + } + } + } + } +#endif + return (description); +#else /* SIOCGIFDESCR */ +get_if_description(const char *name _U_) +{ + return (NULL); +#endif /* SIOCGIFDESCR */ +} + +/* + * Look for a given device in the specified list of devices. + * + * If we find it, return a pointer to its entry. + * + * If we don't find it, attempt to add an entry for it, with the specified + * IFF_ flags and description, and, if that succeeds, return a pointer to + * the new entry, otherwise return NULL and set errbuf to an error message. + */ +pcap_if_t * +find_or_add_if(pcap_if_list_t *devlistp, const char *name, + bpf_u_int32 if_flags, get_if_flags_func get_flags_func, char *errbuf) +{ + bpf_u_int32 pcap_flags; + + /* + * Convert IFF_ flags to pcap flags. + */ + pcap_flags = 0; +#ifdef IFF_LOOPBACK + if (if_flags & IFF_LOOPBACK) + pcap_flags |= PCAP_IF_LOOPBACK; +#else + /* + * We don't have IFF_LOOPBACK, so look at the device name to + * see if it looks like a loopback device. + */ + if (name[0] == 'l' && name[1] == 'o' && + (isdigit((unsigned char)(name[2])) || name[2] == '\0') + pcap_flags |= PCAP_IF_LOOPBACK; +#endif +#ifdef IFF_UP + if (if_flags & IFF_UP) + pcap_flags |= PCAP_IF_UP; +#endif +#ifdef IFF_RUNNING + if (if_flags & IFF_RUNNING) + pcap_flags |= PCAP_IF_RUNNING; +#endif + + /* + * Attempt to find an entry for this device; if we don't find one, + * attempt to add one. + */ + return (find_or_add_dev(devlistp, name, pcap_flags, + get_flags_func, get_if_description(name), errbuf)); +} + +/* + * Look for a given device in the specified list of devices. + * + * If we find it, then, if the specified address isn't null, add it to + * the list of addresses for the device and return 0. + * + * If we don't find it, attempt to add an entry for it, with the specified + * IFF_ flags and description, and, if that succeeds, add the specified + * address to its list of addresses if that address is non-null, and + * return 0, otherwise return -1 and set errbuf to an error message. + * + * (We can get called with a null address because we might get a list + * of interface name/address combinations from the underlying OS, with + * the address being absent in some cases, rather than a list of + * interfaces with each interface having a list of addresses, so this + * call may be the only call made to add to the list, and we want to + * add interfaces even if they have no addresses.) + */ +int +add_addr_to_if(pcap_if_list_t *devlistp, const char *name, + bpf_u_int32 if_flags, get_if_flags_func get_flags_func, + struct sockaddr *addr, size_t addr_size, + struct sockaddr *netmask, size_t netmask_size, + struct sockaddr *broadaddr, size_t broadaddr_size, + struct sockaddr *dstaddr, size_t dstaddr_size, + char *errbuf) +{ + pcap_if_t *curdev; + + /* + * Check whether the device exists and, if not, add it. + */ + curdev = find_or_add_if(devlistp, name, if_flags, get_flags_func, + errbuf); + if (curdev == NULL) { + /* + * Error - give up. + */ + return (-1); + } + + if (addr == NULL) { + /* + * There's no address to add; this entry just meant + * "here's a new interface". + */ + return (0); + } + + /* + * "curdev" is an entry for this interface, and we have an + * address for it; add an entry for that address to the + * interface's list of addresses. + */ + return (add_addr_to_dev(curdev, addr, addr_size, netmask, + netmask_size, broadaddr, broadaddr_size, dstaddr, + dstaddr_size, errbuf)); +} +#endif /* _WIN32 */ + +/* + * Add an entry to the list of addresses for an interface. + * "curdev" is the entry for that interface. + */ +int +add_addr_to_dev(pcap_if_t *curdev, + struct sockaddr *addr, size_t addr_size, + struct sockaddr *netmask, size_t netmask_size, + struct sockaddr *broadaddr, size_t broadaddr_size, + struct sockaddr *dstaddr, size_t dstaddr_size, + char *errbuf) +{ + pcap_addr_t *curaddr, *prevaddr, *nextaddr; + + /* + * Allocate the new entry and fill it in. + */ + curaddr = (pcap_addr_t *)malloc(sizeof(pcap_addr_t)); + if (curaddr == NULL) { + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); + return (-1); + } + + curaddr->next = NULL; + if (addr != NULL && addr_size != 0) { + curaddr->addr = (struct sockaddr *)dup_sockaddr(addr, addr_size); + if (curaddr->addr == NULL) { + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); + free(curaddr); + return (-1); + } + } else + curaddr->addr = NULL; + + if (netmask != NULL && netmask_size != 0) { + curaddr->netmask = (struct sockaddr *)dup_sockaddr(netmask, netmask_size); + if (curaddr->netmask == NULL) { + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); + if (curaddr->addr != NULL) + free(curaddr->addr); + free(curaddr); + return (-1); + } + } else + curaddr->netmask = NULL; + + if (broadaddr != NULL && broadaddr_size != 0) { + curaddr->broadaddr = (struct sockaddr *)dup_sockaddr(broadaddr, broadaddr_size); + if (curaddr->broadaddr == NULL) { + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); + if (curaddr->netmask != NULL) + free(curaddr->netmask); + if (curaddr->addr != NULL) + free(curaddr->addr); + free(curaddr); + return (-1); + } + } else + curaddr->broadaddr = NULL; + + if (dstaddr != NULL && dstaddr_size != 0) { + curaddr->dstaddr = (struct sockaddr *)dup_sockaddr(dstaddr, dstaddr_size); + if (curaddr->dstaddr == NULL) { + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); + if (curaddr->broadaddr != NULL) + free(curaddr->broadaddr); + if (curaddr->netmask != NULL) + free(curaddr->netmask); + if (curaddr->addr != NULL) + free(curaddr->addr); + free(curaddr); return (-1); } + } else + curaddr->dstaddr = NULL; + + /* + * Find the end of the list of addresses. + */ + for (prevaddr = curdev->addresses; prevaddr != NULL; prevaddr = nextaddr) { + nextaddr = prevaddr->next; + if (nextaddr == NULL) { + /* + * This is the end of the list. + */ + break; + } + } + + if (prevaddr == NULL) { + /* + * The list was empty; this is the first member. + */ + curdev->addresses = curaddr; + } else { + /* + * "prevaddr" is the last member of the list; append + * this member to it. + */ + prevaddr->next = curaddr; } return (0); } +/* + * Look for a given device in the specified list of devices. + * + * If we find it, return 0 and set *curdev_ret to point to it. + * + * If we don't find it, attempt to add an entry for it, with the specified + * flags and description, and, if that succeeds, return 0, otherwise + * return -1 and set errbuf to an error message. + */ +pcap_if_t * +find_or_add_dev(pcap_if_list_t *devlistp, const char *name, bpf_u_int32 flags, + get_if_flags_func get_flags_func, const char *description, char *errbuf) +{ + pcap_if_t *curdev; + + /* + * Is there already an entry in the list for this device? + */ + curdev = find_dev(devlistp, name); + if (curdev != NULL) { + /* + * Yes, return it. + */ + return (curdev); + } + + /* + * No, we didn't find it. + */ + + /* + * Try to get additional flags for the device. + */ + if ((*get_flags_func)(name, &flags, errbuf) == -1) { + /* + * Failed. + */ + return (NULL); + } + + /* + * Now, try to add it to the list of devices. + */ + return (add_dev(devlistp, name, flags, description, errbuf)); +} + +/* + * Look for a given device in the specified list of devices, and return + * the entry for it if we find it or NULL if we don't. + */ +pcap_if_t * +find_dev(pcap_if_list_t *devlistp, const char *name) +{ + pcap_if_t *curdev; + + /* + * Is there an entry in the list for this device? + */ + for (curdev = devlistp->beginning; curdev != NULL; + curdev = curdev->next) { + if (strcmp(name, curdev->name) == 0) { + /* + * We found it, so, yes, there is. No need to + * add it. Provide the entry we found to our + * caller. + */ + return (curdev); + } + } + + /* + * No. + */ + return (NULL); +} + +/* + * Attempt to add an entry for a device, with the specified flags + * and description, and, if that succeeds, return 0 and return a pointer + * to the new entry, otherwise return NULL and set errbuf to an error + * message. + * + * If we weren't given a description, try to get one. + */ +pcap_if_t * +add_dev(pcap_if_list_t *devlistp, const char *name, bpf_u_int32 flags, + const char *description, char *errbuf) +{ + pcap_if_t *curdev, *prevdev, *nextdev; + u_int this_figure_of_merit, nextdev_figure_of_merit; + + curdev = malloc(sizeof(pcap_if_t)); + if (curdev == NULL) { + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); + return (NULL); + } + + /* + * Fill in the entry. + */ + curdev->next = NULL; + curdev->name = strdup(name); + if (curdev->name == NULL) { + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); + free(curdev); + return (NULL); + } + if (description == NULL) { + /* + * We weren't handed a description for the interface. + */ + curdev->description = NULL; + } else { + /* + * We were handed a description; make a copy. + */ + curdev->description = strdup(description); + if (curdev->description == NULL) { + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); + free(curdev->name); + free(curdev); + return (NULL); + } + } + curdev->addresses = NULL; /* list starts out as empty */ + curdev->flags = flags; + + /* + * Add it to the list, in the appropriate location. + * First, get the "figure of merit" for this interface. + */ + this_figure_of_merit = get_figure_of_merit(curdev); + + /* + * Now look for the last interface with an figure of merit + * less than or equal to the new interface's figure of merit. + * + * We start with "prevdev" being NULL, meaning we're before + * the first element in the list. + */ + prevdev = NULL; + for (;;) { + /* + * Get the interface after this one. + */ + if (prevdev == NULL) { + /* + * The next element is the first element. + */ + nextdev = devlistp->beginning; + } else + nextdev = prevdev->next; + + /* + * Are we at the end of the list? + */ + if (nextdev == NULL) { + /* + * Yes - we have to put the new entry after "prevdev". + */ + break; + } + + /* + * Is the new interface's figure of merit less + * than the next interface's figure of merit, + * meaning that the new interface is better + * than the next interface? + */ + nextdev_figure_of_merit = get_figure_of_merit(nextdev); + if (this_figure_of_merit < nextdev_figure_of_merit) { + /* + * Yes - we should put the new entry + * before "nextdev", i.e. after "prevdev". + */ + break; + } + + prevdev = nextdev; + } + + /* + * Insert before "nextdev". + */ + curdev->next = nextdev; + + /* + * Insert after "prevdev" - unless "prevdev" is null, + * in which case this is the first interface. + */ + if (prevdev == NULL) { + /* + * This is the first interface. Make it + * the first element in the list of devices. + */ + devlistp->beginning = curdev; + } else + prevdev->next = curdev; + return (curdev); +} + +/* + * Free a list of interfaces. + */ +void +pcap_freealldevs(pcap_if_t *alldevs) +{ + pcap_if_t *curdev, *nextdev; + pcap_addr_t *curaddr, *nextaddr; + + for (curdev = alldevs; curdev != NULL; curdev = nextdev) { + nextdev = curdev->next; + + /* + * Free all addresses. + */ + for (curaddr = curdev->addresses; curaddr != NULL; curaddr = nextaddr) { + nextaddr = curaddr->next; + if (curaddr->addr) + free(curaddr->addr); + if (curaddr->netmask) + free(curaddr->netmask); + if (curaddr->broadaddr) + free(curaddr->broadaddr); + if (curaddr->dstaddr) + free(curaddr->dstaddr); + free(curaddr); + } + + /* + * Free the name string. + */ + free(curdev->name); + + /* + * Free the description string, if any. + */ + if (curdev->description != NULL) + free(curdev->description); + + /* + * Free the interface. + */ + free(curdev); + } +} + +/* + * pcap-npf.c has its own pcap_lookupdev(), for compatibility reasons, as + * it actually returns the names of all interfaces, with a NUL separator + * between them; some callers may depend on that. + * + * MS-DOS has its own pcap_lookupdev(), but that might be useful only + * as an optimization. + * + * In all other cases, we just use pcap_findalldevs() to get a list of + * devices, and pick from that list. + */ +#if !defined(HAVE_PACKET32) && !defined(MSDOS) +/* + * Return the name of a network interface attached to the system, or NULL + * if none can be found. The interface must be configured up; the + * lowest unit number is preferred; loopback is ignored. + */ +char * +pcap_lookupdev(char *errbuf) +{ + pcap_if_t *alldevs; +#ifdef _WIN32 + /* + * Windows - use the same size as the old WinPcap 3.1 code. + * XXX - this is probably bigger than it needs to be. + */ + #define IF_NAMESIZE 8192 +#else + /* + * UN*X - use the system's interface name size. + * XXX - that might not be large enough for capture devices + * that aren't regular network interfaces. + */ + /* for old BSD systems, including bsdi3 */ + #ifndef IF_NAMESIZE + #define IF_NAMESIZE IFNAMSIZ + #endif +#endif + static char device[IF_NAMESIZE + 1]; + char *ret; + + if (pcap_findalldevs(&alldevs, errbuf) == -1) + return (NULL); + + if (alldevs == NULL || (alldevs->flags & PCAP_IF_LOOPBACK)) { + /* + * There are no devices on the list, or the first device + * on the list is a loopback device, which means there + * are no non-loopback devices on the list. This means + * we can't return any device. + * + * XXX - why not return a loopback device? If we can't + * capture on it, it won't be on the list, and if it's + * on the list, there aren't any non-loopback devices, + * so why not just supply it as the default device? + */ + (void)strlcpy(errbuf, "no suitable device found", + PCAP_ERRBUF_SIZE); + ret = NULL; + } else { + /* + * Return the name of the first device on the list. + */ + (void)strlcpy(device, alldevs->name, sizeof(device)); + ret = device; + } + + pcap_freealldevs(alldevs); + return (ret); +} +#endif /* !defined(HAVE_PACKET32) && !defined(MSDOS) */ + +#if !defined(_WIN32) && !defined(MSDOS) +/* + * We don't just fetch the entire list of devices, search for the + * particular device, and use its first IPv4 address, as that's too + * much work to get just one device's netmask. + * + * If we had an API to get attributes for a given device, we could + * use that. + */ +int +pcap_lookupnet(const char *device, bpf_u_int32 *netp, bpf_u_int32 *maskp, + char *errbuf) +{ + register int fd; + register struct sockaddr_in *sin4; + struct ifreq ifr; + + /* + * The pseudo-device "any" listens on all interfaces and therefore + * has the network address and -mask "0.0.0.0" therefore catching + * all traffic. Using NULL for the interface is the same as "any". + */ + if (!device || strcmp(device, "any") == 0 +#ifdef HAVE_DAG_API + || strstr(device, "dag") != NULL +#endif +#ifdef HAVE_SEPTEL_API + || strstr(device, "septel") != NULL +#endif +#ifdef PCAP_SUPPORT_BT + || strstr(device, "bluetooth") != NULL +#endif +#ifdef PCAP_SUPPORT_USB + || strstr(device, "usbmon") != NULL +#endif +#ifdef HAVE_SNF_API + || strstr(device, "snf") != NULL +#endif +#ifdef PCAP_SUPPORT_NETMAP + || strncmp(device, "netmap:", 7) == 0 + || strncmp(device, "vale", 4) == 0 +#endif + ) { + *netp = *maskp = 0; + return 0; + } + + fd = socket(AF_INET, SOCK_DGRAM, 0); + if (fd < 0) { + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "socket"); + return (-1); + } + memset(&ifr, 0, sizeof(ifr)); +#ifdef linux + /* XXX Work around Linux kernel bug */ + ifr.ifr_addr.sa_family = AF_INET; +#endif + (void)strlcpy(ifr.ifr_name, device, sizeof(ifr.ifr_name)); + if (ioctl(fd, SIOCGIFADDR, (char *)&ifr) < 0) { + if (errno == EADDRNOTAVAIL) { + (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, + "%s: no IPv4 address assigned", device); + } else { + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "SIOCGIFADDR: %s", device); + } + (void)close(fd); + return (-1); + } + sin4 = (struct sockaddr_in *)&ifr.ifr_addr; + *netp = sin4->sin_addr.s_addr; + memset(&ifr, 0, sizeof(ifr)); +#ifdef linux + /* XXX Work around Linux kernel bug */ + ifr.ifr_addr.sa_family = AF_INET; +#endif + (void)strlcpy(ifr.ifr_name, device, sizeof(ifr.ifr_name)); + if (ioctl(fd, SIOCGIFNETMASK, (char *)&ifr) < 0) { + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "SIOCGIFNETMASK: %s", device); + (void)close(fd); + return (-1); + } + (void)close(fd); + *maskp = sin4->sin_addr.s_addr; + if (*maskp == 0) { + if (IN_CLASSA(*netp)) + *maskp = IN_CLASSA_NET; + else if (IN_CLASSB(*netp)) + *maskp = IN_CLASSB_NET; + else if (IN_CLASSC(*netp)) + *maskp = IN_CLASSC_NET; + else { + (void)pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, + "inet class for 0x%x unknown", *netp); + return (-1); + } + } + *netp &= *maskp; + return (0); +} +#endif /* !defined(_WIN32) && !defined(MSDOS) */ + +#ifdef ENABLE_REMOTE +#include "pcap-rpcap.h" + +/* + * Extract a substring from a string. + */ +static char * +get_substring(const char *p, size_t len, char *ebuf) +{ + char *token; + + token = malloc(len + 1); + if (token == NULL) { + pcap_fmt_errmsg_for_errno(ebuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); + return (NULL); + } + memcpy(token, p, len); + token[len] = '\0'; + return (token); +} + +/* + * Parse a capture source that might be a URL. + * + * If the source is not a URL, *schemep, *userinfop, *hostp, and *portp + * are set to NULL, *pathp is set to point to the source, and 0 is + * returned. + * + * If source is a URL, and the URL refers to a local device (a special + * case of rpcap:), *schemep, *userinfop, *hostp, and *portp are set + * to NULL, *pathp is set to point to the device name, and 0 is returned. + * + * If source is a URL, and it's not a special case that refers to a local + * device, and the parse succeeds: + * + * *schemep is set to point to an allocated string containing the scheme; + * + * if user information is present in the URL, *userinfop is set to point + * to an allocated string containing the user information, otherwise + * it's set to NULL; + * + * if host information is present in the URL, *hostp is set to point + * to an allocated string containing the host information, otherwise + * it's set to NULL; + * + * if a port number is present in the URL, *portp is set to point + * to an allocated string containing the port number, otherwise + * it's set to NULL; + * + * *pathp is set to point to an allocated string containing the + * path; + * + * and 0 is returned. + * + * If the parse fails, ebuf is set to an error string, and -1 is returned. + */ +static int +pcap_parse_source(const char *source, char **schemep, char **userinfop, + char **hostp, char **portp, char **pathp, char *ebuf) +{ + char *colonp; + size_t scheme_len; + char *scheme; + const char *endp; + size_t authority_len; + char *authority; + char *parsep, *atsignp, *bracketp; + char *userinfo, *host, *port, *path; + + /* + * Start out returning nothing. + */ + *schemep = NULL; + *userinfop = NULL; + *hostp = NULL; + *portp = NULL; + *pathp = NULL; + + /* + * RFC 3986 says: + * + * URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ] + * + * hier-part = "//" authority path-abempty + * / path-absolute + * / path-rootless + * / path-empty + * + * authority = [ userinfo "@" ] host [ ":" port ] + * + * userinfo = *( unreserved / pct-encoded / sub-delims / ":" ) + * + * Step 1: look for the ":" at the end of the scheme. + * A colon in the source is *NOT* sufficient to indicate that + * this is a URL, as interface names on some platforms might + * include colons (e.g., I think some Solaris interfaces + * might). + */ + colonp = strchr(source, ':'); + if (colonp == NULL) { + /* + * The source is the device to open. + * Return a NULL pointer for the scheme, user information, + * host, and port, and return the device as the path. + */ + *pathp = strdup(source); + if (*pathp == NULL) { + pcap_fmt_errmsg_for_errno(ebuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); + return (-1); + } + return (0); + } + + /* + * All schemes must have "//" after them, i.e. we only support + * hier-part = "//" authority path-abempty, not + * hier-part = path-absolute + * hier-part = path-rootless + * hier-part = path-empty + * + * We need that in order to distinguish between a local device + * name that happens to contain a colon and a URI. + */ + if (strncmp(colonp + 1, "//", 2) != 0) { + /* + * The source is the device to open. + * Return a NULL pointer for the scheme, user information, + * host, and port, and return the device as the path. + */ + *pathp = strdup(source); + if (*pathp == NULL) { + pcap_fmt_errmsg_for_errno(ebuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); + return (-1); + } + return (0); + } + + /* + * XXX - check whether the purported scheme could be a scheme? + */ + + /* + * OK, this looks like a URL. + * Get the scheme. + */ + scheme_len = colonp - source; + scheme = malloc(scheme_len + 1); + if (scheme == NULL) { + pcap_fmt_errmsg_for_errno(ebuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); + return (-1); + } + memcpy(scheme, source, scheme_len); + scheme[scheme_len] = '\0'; + + /* + * Treat file: specially - take everything after file:// as + * the pathname. + */ + if (pcap_strcasecmp(scheme, "file") == 0) { + *schemep = scheme; + *pathp = strdup(colonp + 3); + if (*pathp == NULL) { + pcap_fmt_errmsg_for_errno(ebuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); + return (-1); + } + return (0); + } + + /* + * The WinPcap documentation says you can specify a local + * interface with "rpcap://{device}"; we special-case + * that here. If the scheme is "rpcap", and there are + * no slashes past the "//", we just return the device. + * + * XXX - %-escaping? + */ + if (pcap_strcasecmp(scheme, "rpcap") == 0 && + strchr(colonp + 3, '/') == NULL) { + /* + * Local device. + * + * Return a NULL pointer for the scheme, user information, + * host, and port, and return the device as the path. + */ + free(scheme); + *pathp = strdup(colonp + 3); + if (*pathp == NULL) { + pcap_fmt_errmsg_for_errno(ebuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); + return (-1); + } + return (0); + } + + /* + * OK, now start parsing the authority. + * Get token, terminated with / or terminated at the end of + * the string. + */ + authority_len = strcspn(colonp + 3, "/"); + authority = get_substring(colonp + 3, authority_len, ebuf); + if (authority == NULL) { + /* + * Error. + */ + free(scheme); + return (-1); + } + endp = colonp + 3 + authority_len; + + /* + * Now carve the authority field into its components. + */ + parsep = authority; + + /* + * Is there a userinfo field? + */ + atsignp = strchr(parsep, '@'); + if (atsignp != NULL) { + /* + * Yes. + */ + size_t userinfo_len; + + userinfo_len = atsignp - parsep; + userinfo = get_substring(parsep, userinfo_len, ebuf); + if (userinfo == NULL) { + /* + * Error. + */ + free(authority); + free(scheme); + return (-1); + } + parsep = atsignp + 1; + } else { + /* + * No. + */ + userinfo = NULL; + } + + /* + * Is there a host field? + */ + if (*parsep == '\0') { + /* + * No; there's no host field or port field. + */ + host = NULL; + port = NULL; + } else { + /* + * Yes. + */ + size_t host_len; + + /* + * Is it an IP-literal? + */ + if (*parsep == '[') { + /* + * Yes. + * Treat verything up to the closing square + * bracket as the IP-Literal; we don't worry + * about whether it's a valid IPv6address or + * IPvFuture. + */ + bracketp = strchr(parsep, ']'); + if (bracketp == NULL) { + /* + * There's no closing square bracket. + */ + pcap_snprintf(ebuf, PCAP_ERRBUF_SIZE, + "IP-literal in URL doesn't end with ]"); + free(userinfo); + free(authority); + free(scheme); + return (-1); + } + if (*(bracketp + 1) != '\0' && + *(bracketp + 1) != ':') { + /* + * There's extra crud after the + * closing square bracketn. + */ + pcap_snprintf(ebuf, PCAP_ERRBUF_SIZE, + "Extra text after IP-literal in URL"); + free(userinfo); + free(authority); + free(scheme); + return (-1); + } + host_len = (bracketp - 1) - parsep; + host = get_substring(parsep + 1, host_len, ebuf); + if (host == NULL) { + /* + * Error. + */ + free(userinfo); + free(authority); + free(scheme); + return (-1); + } + parsep = bracketp + 1; + } else { + /* + * No. + * Treat everything up to a : or the end of + * the string as the host. + */ + host_len = strcspn(parsep, ":"); + host = get_substring(parsep, host_len, ebuf); + if (host == NULL) { + /* + * Error. + */ + free(userinfo); + free(authority); + free(scheme); + return (-1); + } + parsep = parsep + host_len; + } + + /* + * Is there a port field? + */ + if (*parsep == ':') { + /* + * Yes. It's the rest of the authority field. + */ + size_t port_len; + + parsep++; + port_len = strlen(parsep); + port = get_substring(parsep, port_len, ebuf); + if (port == NULL) { + /* + * Error. + */ + free(host); + free(userinfo); + free(authority); + free(scheme); + return (-1); + } + } else { + /* + * No. + */ + port = NULL; + } + } + free(authority); + + /* + * Everything else is the path. Strip off the leading /. + */ + if (*endp == '\0') + path = strdup(""); + else + path = strdup(endp + 1); + if (path == NULL) { + pcap_fmt_errmsg_for_errno(ebuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); + free(port); + free(host); + free(userinfo); + free(scheme); + return (-1); + } + *schemep = scheme; + *userinfop = userinfo; + *hostp = host; + *portp = port; + *pathp = path; + return (0); +} + +int +pcap_createsrcstr(char *source, int type, const char *host, const char *port, + const char *name, char *errbuf) +{ + switch (type) { + + case PCAP_SRC_FILE: + strlcpy(source, PCAP_SRC_FILE_STRING, PCAP_BUF_SIZE); + if (name != NULL && *name != '\0') { + strlcat(source, name, PCAP_BUF_SIZE); + return (0); + } else { + pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, + "The file name cannot be NULL."); + return (-1); + } + + case PCAP_SRC_IFREMOTE: + strlcpy(source, PCAP_SRC_IF_STRING, PCAP_BUF_SIZE); + if (host != NULL && *host != '\0') { + if (strchr(host, ':') != NULL) { + /* + * The host name contains a colon, so it's + * probably an IPv6 address, and needs to + * be included in square brackets. + */ + strlcat(source, "[", PCAP_BUF_SIZE); + strlcat(source, host, PCAP_BUF_SIZE); + strlcat(source, "]", PCAP_BUF_SIZE); + } else + strlcat(source, host, PCAP_BUF_SIZE); + + if (port != NULL && *port != '\0') { + strlcat(source, ":", PCAP_BUF_SIZE); + strlcat(source, port, PCAP_BUF_SIZE); + } + + strlcat(source, "/", PCAP_BUF_SIZE); + } else { + pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, + "The host name cannot be NULL."); + return (-1); + } + + if (name != NULL && *name != '\0') + strlcat(source, name, PCAP_BUF_SIZE); + + return (0); + + case PCAP_SRC_IFLOCAL: + strlcpy(source, PCAP_SRC_IF_STRING, PCAP_BUF_SIZE); + + if (name != NULL && *name != '\0') + strlcat(source, name, PCAP_BUF_SIZE); + + return (0); + + default: + pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, + "The interface type is not valid."); + return (-1); + } +} + +int +pcap_parsesrcstr(const char *source, int *type, char *host, char *port, + char *name, char *errbuf) +{ + char *scheme, *tmpuserinfo, *tmphost, *tmpport, *tmppath; + + /* Initialization stuff */ + if (host) + *host = '\0'; + if (port) + *port = '\0'; + if (name) + *name = '\0'; + + /* Parse the source string */ + if (pcap_parse_source(source, &scheme, &tmpuserinfo, &tmphost, + &tmpport, &tmppath, errbuf) == -1) { + /* + * Fail. + */ + return (-1); + } + + if (scheme == NULL) { + /* + * Local device. + */ + if (name && tmppath) + strlcpy(name, tmppath, PCAP_BUF_SIZE); + if (type) + *type = PCAP_SRC_IFLOCAL; + free(tmppath); + free(tmpport); + free(tmphost); + free(tmpuserinfo); + return (0); + } + + if (strcmp(scheme, "rpcap") == 0) { + /* + * rpcap:// + * + * pcap_parse_source() has already handled the case of + * rpcap://device + */ + if (host && tmphost) { + if (tmpuserinfo) + pcap_snprintf(host, PCAP_BUF_SIZE, "%s@%s", + tmpuserinfo, tmphost); + else + strlcpy(host, tmphost, PCAP_BUF_SIZE); + } + if (port && tmpport) + strlcpy(port, tmpport, PCAP_BUF_SIZE); + if (name && tmppath) + strlcpy(name, tmppath, PCAP_BUF_SIZE); + if (type) + *type = PCAP_SRC_IFREMOTE; + free(tmppath); + free(tmpport); + free(tmphost); + free(tmpuserinfo); + free(scheme); + return (0); + } + + if (strcmp(scheme, "file") == 0) { + /* + * file:// + */ + if (name && tmppath) + strlcpy(name, tmppath, PCAP_BUF_SIZE); + if (type) + *type = PCAP_SRC_FILE; + free(tmppath); + free(tmpport); + free(tmphost); + free(tmpuserinfo); + free(scheme); + return (0); + } + + /* + * Neither rpcap: nor file:; just treat the entire string + * as a local device. + */ + if (name) + strlcpy(name, source, PCAP_BUF_SIZE); + if (type) + *type = PCAP_SRC_IFLOCAL; + free(tmppath); + free(tmpport); + free(tmphost); + free(tmpuserinfo); + free(scheme); + return (0); +} +#endif + pcap_t * pcap_create(const char *device, char *errbuf) { @@ -410,8 +1995,9 @@ pcap_create(const char *device, char *errbuf) length = wcslen((wchar_t *)device); device_str = (char *)malloc(length + 1); if (device_str == NULL) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "malloc: %s", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, + PCAP_ERRBUF_SIZE, errno, + "malloc"); return (NULL); } @@ -422,8 +2008,8 @@ pcap_create(const char *device, char *errbuf) device_str = strdup(device); } if (device_str == NULL) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "malloc: %s", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); return (NULL); } @@ -473,6 +2059,18 @@ pcap_create(const char *device, char *errbuf) return (p); } +/* + * Set nonblocking mode on an unactivated pcap_t; this sets a flag + * checked by pcap_activate(), which sets the mode after calling + * the activate routine. + */ +static int +pcap_setnonblock_unactivated(pcap_t *p, int nonblock) +{ + p->opt.nonblock = nonblock; + return (0); +} + static void initialize_ops(pcap_t *p) { @@ -487,7 +2085,6 @@ initialize_ops(pcap_t *p) p->setdirection_op = (setdirection_op_t)pcap_not_initialized; p->set_datalink_op = (set_datalink_op_t)pcap_not_initialized; p->getnonblock_op = (getnonblock_op_t)pcap_not_initialized; - p->setnonblock_op = (setnonblock_op_t)pcap_not_initialized; p->stats_op = (stats_op_t)pcap_not_initialized; #ifdef _WIN32 p->stats_ex_op = (stats_ex_op_t)pcap_not_initialized_ptr; @@ -532,8 +2129,8 @@ pcap_alloc_pcap_t(char *ebuf, size_t size) */ chunk = malloc(sizeof (pcap_t) + size); if (chunk == NULL) { - pcap_snprintf(ebuf, PCAP_ERRBUF_SIZE, "malloc: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(ebuf, PCAP_ERRBUF_SIZE, + errno, "malloc"); return (NULL); } memset(chunk, 0, sizeof (pcap_t) + size); @@ -543,10 +2140,15 @@ pcap_alloc_pcap_t(char *ebuf, size_t size) */ p = (pcap_t *)chunk; -#ifndef _WIN32 +#ifdef _WIN32 + p->handle = INVALID_HANDLE_VALUE; /* not opened yet */ +#else /* _WIN32 */ p->fd = -1; /* not opened yet */ +#ifndef MSDOS p->selectable_fd = -1; -#endif + p->required_select_timeout = NULL; +#endif /* MSDOS */ +#endif /* _WIN32 */ if (size == 0) { /* No private data was requested. */ @@ -579,10 +2181,17 @@ pcap_create_common(char *ebuf, size_t size) */ p->can_set_rfmon_op = pcap_cant_set_rfmon; + /* + * If pcap_setnonblock() is called on a not-yet-activated + * pcap_t, default to setting a flag and turning + * on non-blocking mode when activated. + */ + p->setnonblock_op = pcap_setnonblock_unactivated; + initialize_ops(p); /* put in some defaults*/ - p->snapshot = MAXIMUM_SNAPLEN; /* max packet size */ + p->snapshot = 0; /* max packet size unspecified */ p->opt.timeout = 0; /* no timeout specified */ p->opt.buffer_size = 0; /* use the platform's default */ p->opt.promisc = 0; @@ -590,6 +2199,15 @@ pcap_create_common(char *ebuf, size_t size) p->opt.immediate = 0; p->opt.tstamp_type = -1; /* default to not setting time stamp type */ p->opt.tstamp_precision = PCAP_TSTAMP_PRECISION_MICRO; + /* + * Platform-dependent options. + */ +#ifdef __linux__ + p->opt.protocol = 0; +#endif +#ifdef _WIN32 + p->opt.nocapture_local = 0; +#endif /* * Start out with no BPF code generation flags set. @@ -615,16 +2233,6 @@ pcap_set_snaplen(pcap_t *p, int snaplen) { if (pcap_check_activated(p)) return (PCAP_ERROR_ACTIVATED); - - /* - * Turn invalid values, or excessively large values, into - * the maximum allowed value. - * - * If some application really *needs* a bigger snapshot - * length, we should just increase MAXIMUM_SNAPLEN. - */ - if (snaplen <= 0 || snaplen > MAXIMUM_SNAPLEN) - snaplen = MAXIMUM_SNAPLEN; p->snapshot = snaplen; return (0); } @@ -795,9 +2403,25 @@ pcap_activate(pcap_t *p) if (pcap_check_activated(p)) return (PCAP_ERROR_ACTIVATED); status = p->activate_op(p); - if (status >= 0) + if (status >= 0) { + /* + * If somebody requested non-blocking mode before + * calling pcap_activate(), turn it on now. + */ + if (p->opt.nonblock) { + status = p->setnonblock_op(p, 1); + if (status < 0) { + /* + * Failed. Undo everything done by + * the activate operation. + */ + p->cleanup_op(p); + initialize_ops(p); + return (status); + } + } p->activated = 1; - else { + } else { if (p->errbuf[0] == '\0') { /* * No error message supplied by the activate routine; @@ -823,6 +2447,53 @@ pcap_open_live(const char *device, int snaplen, int promisc, int to_ms, char *er { pcap_t *p; int status; +#ifdef ENABLE_REMOTE + char host[PCAP_BUF_SIZE + 1]; + char port[PCAP_BUF_SIZE + 1]; + char name[PCAP_BUF_SIZE + 1]; + int srctype; + + /* + * Retrofit - we have to make older applications compatible with + * remote capture. + * So we're calling pcap_open_remote() from here; this is a very + * dirty hack. + * Obviously, we cannot exploit all the new features; for instance, + * we cannot send authentication, we cannot use a UDP data connection, + * and so on. + */ + if (pcap_parsesrcstr(device, &srctype, host, port, name, errbuf)) + return (NULL); + + if (srctype == PCAP_SRC_IFREMOTE) { + /* + * Although we already have host, port and iface, we prefer + * to pass only 'device' to pcap_open_rpcap(), so that it has + * to call pcap_parsesrcstr() again. + * This is less optimized, but much clearer. + */ + return (pcap_open_rpcap(device, snaplen, + promisc ? PCAP_OPENFLAG_PROMISCUOUS : 0, to_ms, + NULL, errbuf)); + } + if (srctype == PCAP_SRC_FILE) { + pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, "unknown URL scheme \"file\""); + return (NULL); + } + if (srctype == PCAP_SRC_IFLOCAL) { + /* + * If it starts with rpcap://, that refers to a local device + * (no host part in the URL). Remove the rpcap://, and + * fall through to the regular open path. + */ + if (strncmp(device, PCAP_SRC_IF_STRING, strlen(PCAP_SRC_IF_STRING)) == 0) { + size_t len = strlen(device) - strlen(PCAP_SRC_IF_STRING) + 1; + + if (len > 0) + device += strlen(PCAP_SRC_IF_STRING); + } + } +#endif /* ENABLE_REMOTE */ p = pcap_create(device, errbuf); if (p == NULL) @@ -956,8 +2627,8 @@ pcap_list_datalinks(pcap_t *p, int **dlt_buffer) */ *dlt_buffer = (int*)malloc(sizeof(**dlt_buffer)); if (*dlt_buffer == NULL) { - (void)pcap_snprintf(p->errbuf, sizeof(p->errbuf), - "malloc: %s", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, sizeof(p->errbuf), + errno, "malloc"); return (PCAP_ERROR); } **dlt_buffer = p->linktype; @@ -965,8 +2636,8 @@ pcap_list_datalinks(pcap_t *p, int **dlt_buffer) } else { *dlt_buffer = (int*)calloc(sizeof(**dlt_buffer), p->dlt_count); if (*dlt_buffer == NULL) { - (void)pcap_snprintf(p->errbuf, sizeof(p->errbuf), - "malloc: %s", pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, sizeof(p->errbuf), + errno, "malloc"); return (PCAP_ERROR); } (void)memcpy(*dlt_buffer, p->dlt_list, @@ -1281,6 +2952,15 @@ static struct dlt_choice dlt_choices[] = { DLT_CHOICE(WATTSTOPPER_DLM, "WattStopper Digital Lighting Management (DLM) and Legrand Nitoo Open protocol"), DLT_CHOICE(ISO_14443, "ISO 14443 messages"), DLT_CHOICE(RDS, "IEC 62106 Radio Data System groups"), + DLT_CHOICE(USB_DARWIN, "USB with Darwin header"), + DLT_CHOICE(OPENFLOW, "OpenBSD DLT_OPENFLOW"), + DLT_CHOICE(SDLC, "IBM SDLC frames"), + DLT_CHOICE(TI_LLN_SNIFFER, "TI LLN sniffer frames"), + DLT_CHOICE(VSOCK, "Linux vsock"), + DLT_CHOICE(NORDIC_BLE, "Nordic Semiconductor Bluetooth LE sniffer frames"), + DLT_CHOICE(DOCSIS31_XRA31, "Excentis XRA-31 DOCSIS 3.1 RF sniffer frames"), + DLT_CHOICE(ETHERNET_MPACKET, "802.3br mPackets"), + DLT_CHOICE(DISPLAYPORT_AUX, "DisplayPort AUX channel monitoring data"), DLT_CHOICE_SENTINEL }; @@ -1403,6 +3083,14 @@ pcap_minor_version(pcap_t *p) return (p->version_minor); } +int +pcap_bufsize(pcap_t *p) +{ + if (!p->activated) + return (PCAP_ERROR_NOT_ACTIVATED); + return (p->bufsize); +} + FILE * pcap_file(pcap_t *p) { @@ -1415,8 +3103,8 @@ pcap_fileno(pcap_t *p) #ifndef _WIN32 return (p->fd); #else - if (p->adapter != NULL) - return ((int)(DWORD)p->adapter->hFile); + if (p->handle != INVALID_HANDLE_VALUE) + return ((int)(DWORD)p->handle); else return (PCAP_ERROR); #endif @@ -1428,6 +3116,12 @@ pcap_get_selectable_fd(pcap_t *p) { return (p->selectable_fd); } + +struct timeval * +pcap_get_required_select_timeout(pcap_t *p) +{ + return (p->required_select_timeout); +} #endif void @@ -1447,14 +3141,18 @@ pcap_getnonblock(pcap_t *p, char *errbuf) { int ret; - ret = p->getnonblock_op(p, errbuf); + ret = p->getnonblock_op(p); if (ret == -1) { /* - * In case somebody depended on the bug wherein - * the error message was put into p->errbuf - * by pcap_getnonblock_fd(). + * The get nonblock operation sets p->errbuf; this + * function *shouldn't* have had a separate errbuf + * argument, as it didn't need one, but I goofed + * when adding it. + * + * We copy the error message to errbuf, so callers + * can find it in either place. */ - strlcpy(p->errbuf, errbuf, PCAP_ERRBUF_SIZE); + strlcpy(errbuf, p->errbuf, PCAP_ERRBUF_SIZE); } return (ret); } @@ -1465,14 +3163,14 @@ pcap_getnonblock(pcap_t *p, char *errbuf) */ #if !defined(_WIN32) && !defined(MSDOS) int -pcap_getnonblock_fd(pcap_t *p, char *errbuf) +pcap_getnonblock_fd(pcap_t *p) { int fdflags; fdflags = fcntl(p->fd, F_GETFL, 0); if (fdflags == -1) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, "F_GETFL: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "F_GETFL"); return (-1); } if (fdflags & O_NONBLOCK) @@ -1487,14 +3185,18 @@ pcap_setnonblock(pcap_t *p, int nonblock, char *errbuf) { int ret; - ret = p->setnonblock_op(p, nonblock, errbuf); + ret = p->setnonblock_op(p, nonblock); if (ret == -1) { /* - * In case somebody depended on the bug wherein - * the error message was put into p->errbuf - * by pcap_setnonblock_fd(). + * The set nonblock operation sets p->errbuf; this + * function *shouldn't* have had a separate errbuf + * argument, as it didn't need one, but I goofed + * when adding it. + * + * We copy the error message to errbuf, so callers + * can find it in either place. */ - strlcpy(p->errbuf, errbuf, PCAP_ERRBUF_SIZE); + strlcpy(errbuf, p->errbuf, PCAP_ERRBUF_SIZE); } return (ret); } @@ -1507,14 +3209,14 @@ pcap_setnonblock(pcap_t *p, int nonblock, char *errbuf) * needs to do some additional work.) */ int -pcap_setnonblock_fd(pcap_t *p, int nonblock, char *errbuf) +pcap_setnonblock_fd(pcap_t *p, int nonblock) { int fdflags; fdflags = fcntl(p->fd, F_GETFL, 0); if (fdflags == -1) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, "F_GETFL: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "F_GETFL"); return (-1); } if (nonblock) @@ -1522,8 +3224,8 @@ pcap_setnonblock_fd(pcap_t *p, int nonblock, char *errbuf) else fdflags &= ~O_NONBLOCK; if (fcntl(p->fd, F_SETFL, fdflags) == -1) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, "F_SETFL: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "F_SETFL"); return (-1); } return (0); @@ -1627,9 +3329,9 @@ pcap_strerror(int errnum) #ifdef HAVE_STRERROR #ifdef _WIN32 static char errbuf[PCAP_ERRBUF_SIZE]; - errno_t errno; - errno = strerror_s(errbuf, PCAP_ERRBUF_SIZE, errnum); - if (errno != 0) /* errno = 0 if successful */ + errno_t err = strerror_s(errbuf, PCAP_ERRBUF_SIZE, errnum); + + if (err != 0) /* err = 0 if successful */ strlcpy(errbuf, "strerror_s() error", PCAP_ERRBUF_SIZE); return (errbuf); #else @@ -1676,14 +3378,6 @@ pcap_stats(pcap_t *p, struct pcap_stat *ps) return (p->stats_op(p, ps)); } -static int -pcap_stats_dead(pcap_t *p, struct pcap_stat *ps _U_) -{ - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "Statistics aren't available from a pcap_open_dead pcap_t"); - return (-1); -} - #ifdef _WIN32 struct pcap_stat * pcap_stats_ex(pcap_t *p, int *pcap_stat_size) @@ -1697,86 +3391,36 @@ pcap_setbuff(pcap_t *p, int dim) return (p->setbuff_op(p, dim)); } -static int -pcap_setbuff_dead(pcap_t *p, int dim) -{ - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "The kernel buffer size cannot be set on a pcap_open_dead pcap_t"); - return (-1); -} - int pcap_setmode(pcap_t *p, int mode) { return (p->setmode_op(p, mode)); } -static int -pcap_setmode_dead(pcap_t *p, int mode) -{ - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "impossible to set mode on a pcap_open_dead pcap_t"); - return (-1); -} - int pcap_setmintocopy(pcap_t *p, int size) { return (p->setmintocopy_op(p, size)); } -static int -pcap_setmintocopy_dead(pcap_t *p, int size) -{ - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "The mintocopy parameter cannot be set on a pcap_open_dead pcap_t"); - return (-1); -} - HANDLE pcap_getevent(pcap_t *p) { return (p->getevent_op(p)); } -static HANDLE -pcap_getevent_dead(pcap_t *p) -{ - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "A pcap_open_dead pcap_t has no event handle"); - return (INVALID_HANDLE_VALUE); -} - int pcap_oid_get_request(pcap_t *p, bpf_u_int32 oid, void *data, size_t *lenp) { return (p->oid_get_request_op(p, oid, data, lenp)); } -static int -pcap_oid_get_request_dead(pcap_t *p, bpf_u_int32 oid _U_, void *data _U_, - size_t *lenp _U_) -{ - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "An OID get request cannot be performed on a pcap_open_dead pcap_t"); - return (PCAP_ERROR); -} - int pcap_oid_set_request(pcap_t *p, bpf_u_int32 oid, const void *data, size_t *lenp) { return (p->oid_set_request_op(p, oid, data, lenp)); } -static int -pcap_oid_set_request_dead(pcap_t *p, bpf_u_int32 oid _U_, const void *data _U_, - size_t *lenp _U_) -{ - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "An OID set request cannot be performed on a pcap_open_dead pcap_t"); - return (PCAP_ERROR); -} - pcap_send_queue * pcap_sendqueue_alloc(u_int memsize) { @@ -1832,56 +3476,24 @@ pcap_sendqueue_transmit(pcap_t *p, pcap_send_queue *queue, int sync) return (p->sendqueue_transmit_op(p, queue, sync)); } -static u_int -pcap_sendqueue_transmit_dead(pcap_t *p, pcap_send_queue *queue, int sync) -{ - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "Packets cannot be transmitted on a pcap_open_dead pcap_t"); - return (0); -} - int pcap_setuserbuffer(pcap_t *p, int size) { return (p->setuserbuffer_op(p, size)); } -static int -pcap_setuserbuffer_dead(pcap_t *p, int size) -{ - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "The user buffer cannot be set on a pcap_open_dead pcap_t"); - return (-1); -} - int pcap_live_dump(pcap_t *p, char *filename, int maxsize, int maxpacks) { return (p->live_dump_op(p, filename, maxsize, maxpacks)); } -static int -pcap_live_dump_dead(pcap_t *p, char *filename, int maxsize, int maxpacks) -{ - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "Live packet dumping cannot be performed on a pcap_open_dead pcap_t"); - return (-1); -} - int pcap_live_dump_ended(pcap_t *p, int sync) { return (p->live_dump_ended_op(p, sync)); } -static int -pcap_live_dump_ended_dead(pcap_t *p, int sync) -{ - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "Live packet dumping cannot be performed on a pcap_open_dead pcap_t"); - return (-1); -} - PAirpcapHandle pcap_get_airpcap_handle(pcap_t *p) { @@ -1894,12 +3506,6 @@ pcap_get_airpcap_handle(pcap_t *p) } return (handle); } - -static PAirpcapHandle -pcap_get_airpcap_handle_dead(pcap_t *p) -{ - return (NULL); -} #endif /* @@ -2022,66 +3628,6 @@ pcap_cleanup_live_common(pcap_t *p) #endif } -static void -pcap_cleanup_dead(pcap_t *p _U_) -{ - /* Nothing to do. */ -} - -pcap_t * -pcap_open_dead_with_tstamp_precision(int linktype, int snaplen, u_int precision) -{ - pcap_t *p; - - switch (precision) { - - case PCAP_TSTAMP_PRECISION_MICRO: - case PCAP_TSTAMP_PRECISION_NANO: - break; - - default: - return NULL; - } - p = malloc(sizeof(*p)); - if (p == NULL) - return NULL; - memset (p, 0, sizeof(*p)); - p->snapshot = snaplen; - p->linktype = linktype; - p->opt.tstamp_precision = precision; - p->stats_op = pcap_stats_dead; -#ifdef _WIN32 - p->stats_ex_op = (stats_ex_op_t)pcap_not_initialized_ptr; - p->setbuff_op = pcap_setbuff_dead; - p->setmode_op = pcap_setmode_dead; - p->setmintocopy_op = pcap_setmintocopy_dead; - p->getevent_op = pcap_getevent_dead; - p->oid_get_request_op = pcap_oid_get_request_dead; - p->oid_set_request_op = pcap_oid_set_request_dead; - p->sendqueue_transmit_op = pcap_sendqueue_transmit_dead; - p->setuserbuffer_op = pcap_setuserbuffer_dead; - p->live_dump_op = pcap_live_dump_dead; - p->live_dump_ended_op = pcap_live_dump_ended_dead; - p->get_airpcap_handle_op = pcap_get_airpcap_handle_dead; -#endif - p->cleanup_op = pcap_cleanup_dead; - - /* - * A "dead" pcap_t never requires special BPF code generation. - */ - p->bpf_codegen_flags = 0; - - p->activated = 1; - return (p); -} - -pcap_t * -pcap_open_dead(int linktype, int snaplen) -{ - return (pcap_open_dead_with_tstamp_precision(linktype, snaplen, - PCAP_TSTAMP_PRECISION_MICRO)); -} - /* * API compatible with WinPcap's "send a packet" routine - returns -1 * on error, 0 otherwise. @@ -2133,160 +3679,251 @@ pcap_offline_filter(const struct bpf_program *fp, const struct pcap_pkthdr *h, return (0); } -#include "pcap_version.h" +static int +pcap_can_set_rfmon_dead(pcap_t *p) +{ + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "Rfmon mode doesn't apply on a pcap_open_dead pcap_t"); + return (PCAP_ERROR); +} + +static int +pcap_read_dead(pcap_t *p, int cnt _U_, pcap_handler callback _U_, + u_char *user _U_) +{ + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "Packets aren't available from a pcap_open_dead pcap_t"); + return (-1); +} + +static int +pcap_inject_dead(pcap_t *p, const void *buf _U_, size_t size _U_) +{ + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "Packets can't be sent on a pcap_open_dead pcap_t"); + return (-1); +} + +static int +pcap_setfilter_dead(pcap_t *p, struct bpf_program *fp _U_) +{ + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "A filter cannot be set on a pcap_open_dead pcap_t"); + return (-1); +} + +static int +pcap_setdirection_dead(pcap_t *p, pcap_direction_t d _U_) +{ + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "The packet direction cannot be set on a pcap_open_dead pcap_t"); + return (-1); +} + +static int +pcap_set_datalink_dead(pcap_t *p, int dlt _U_) +{ + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "The link-layer header type cannot be set on a pcap_open_dead pcap_t"); + return (-1); +} + +static int +pcap_getnonblock_dead(pcap_t *p) +{ + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "A pcap_open_dead pcap_t does not have a non-blocking mode setting"); + return (-1); +} + +static int +pcap_setnonblock_dead(pcap_t *p, int nonblock _U_) +{ + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "A pcap_open_dead pcap_t does not have a non-blocking mode setting"); + return (-1); +} + +static int +pcap_stats_dead(pcap_t *p, struct pcap_stat *ps _U_) +{ + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "Statistics aren't available from a pcap_open_dead pcap_t"); + return (-1); +} #ifdef _WIN32 +struct pcap_stat * +pcap_stats_ex_dead(pcap_t *p, int *pcap_stat_size _U_) +{ + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "Statistics aren't available from a pcap_open_dead pcap_t"); + return (NULL); +} -static char *full_pcap_version_string; +static int +pcap_setbuff_dead(pcap_t *p, int dim) +{ + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "The kernel buffer size cannot be set on a pcap_open_dead pcap_t"); + return (-1); +} -#ifdef HAVE_VERSION_H -/* - * libpcap being built for Windows, as part of a WinPcap/Npcap source - * tree. Include version.h from that source tree to get the WinPcap/Npcap - * version. - * - * XXX - it'd be nice if we could somehow generate the WinPcap version number - * when building WinPcap. (It'd be nice to do so for the packet.dll version - * number as well.) - */ -#include "../../version.h" +static int +pcap_setmode_dead(pcap_t *p, int mode) +{ + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "impossible to set mode on a pcap_open_dead pcap_t"); + return (-1); +} -static const char wpcap_version_string[] = WINPCAP_VER_STRING; -static const char pcap_version_string_fmt[] = - WINPCAP_PRODUCT_NAME " version %s, based on %s"; -static const char pcap_version_string_packet_dll_fmt[] = - WINPCAP_PRODUCT_NAME " version %s (packet.dll version %s), based on %s"; +static int +pcap_setmintocopy_dead(pcap_t *p, int size) +{ + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "The mintocopy parameter cannot be set on a pcap_open_dead pcap_t"); + return (-1); +} -const char * -pcap_lib_version(void) +static HANDLE +pcap_getevent_dead(pcap_t *p) { - char *packet_version_string; - size_t full_pcap_version_string_len; + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "A pcap_open_dead pcap_t has no event handle"); + return (INVALID_HANDLE_VALUE); +} - if (full_pcap_version_string == NULL) { - /* - * Generate the version string. - */ - packet_version_string = PacketGetVersion(); - if (strcmp(wpcap_version_string, packet_version_string) == 0) { - /* - * WinPcap version string and packet.dll version - * string are the same; just report the WinPcap - * version. - */ - full_pcap_version_string_len = - (sizeof pcap_version_string_fmt - 4) + - strlen(wpcap_version_string) + - strlen(pcap_version_string); - full_pcap_version_string = - malloc(full_pcap_version_string_len); - if (full_pcap_version_string == NULL) - return (NULL); - pcap_snprintf(full_pcap_version_string, - full_pcap_version_string_len, - pcap_version_string_fmt, - wpcap_version_string, - pcap_version_string); - } else { - /* - * WinPcap version string and packet.dll version - * string are different; that shouldn't be the - * case (the two libraries should come from the - * same version of WinPcap), so we report both - * versions. - */ - full_pcap_version_string_len = - (sizeof pcap_version_string_packet_dll_fmt - 6) + - strlen(wpcap_version_string) + - strlen(packet_version_string) + - strlen(pcap_version_string); - full_pcap_version_string = malloc(full_pcap_version_string_len); - if (full_pcap_version_string == NULL) - return (NULL); - pcap_snprintf(full_pcap_version_string, - full_pcap_version_string_len, - pcap_version_string_packet_dll_fmt, - wpcap_version_string, - packet_version_string, - pcap_version_string); - } - } - return (full_pcap_version_string); +static int +pcap_oid_get_request_dead(pcap_t *p, bpf_u_int32 oid _U_, void *data _U_, + size_t *lenp _U_) +{ + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "An OID get request cannot be performed on a pcap_open_dead pcap_t"); + return (PCAP_ERROR); } -#else /* HAVE_VERSION_H */ +static int +pcap_oid_set_request_dead(pcap_t *p, bpf_u_int32 oid _U_, const void *data _U_, + size_t *lenp _U_) +{ + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "An OID set request cannot be performed on a pcap_open_dead pcap_t"); + return (PCAP_ERROR); +} -/* - * libpcap being built for Windows, not as part of a WinPcap/Npcap source - * tree. - */ -static const char pcap_version_string_packet_dll_fmt[] = - "%s (packet.dll version %s)"; -const char * -pcap_lib_version(void) +static u_int +pcap_sendqueue_transmit_dead(pcap_t *p, pcap_send_queue *queue, int sync) { - char *packet_version_string; - size_t full_pcap_version_string_len; + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "Packets cannot be transmitted on a pcap_open_dead pcap_t"); + return (0); +} - if (full_pcap_version_string == NULL) { - /* - * Generate the version string. Report the packet.dll - * version. - */ - packet_version_string = PacketGetVersion(); - full_pcap_version_string_len = - (sizeof pcap_version_string_packet_dll_fmt - 4) + - strlen(pcap_version_string) + - strlen(packet_version_string); - full_pcap_version_string = malloc(full_pcap_version_string_len); - if (full_pcap_version_string == NULL) - return (NULL); - pcap_snprintf(full_pcap_version_string, - full_pcap_version_string_len, - pcap_version_string_packet_dll_fmt, - pcap_version_string, - packet_version_string); - } - return (full_pcap_version_string); +static int +pcap_setuserbuffer_dead(pcap_t *p, int size) +{ + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "The user buffer cannot be set on a pcap_open_dead pcap_t"); + return (-1); } -#endif /* HAVE_VERSION_H */ +static int +pcap_live_dump_dead(pcap_t *p, char *filename, int maxsize, int maxpacks) +{ + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "Live packet dumping cannot be performed on a pcap_open_dead pcap_t"); + return (-1); +} -#elif defined(MSDOS) +static int +pcap_live_dump_ended_dead(pcap_t *p, int sync) +{ + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "Live packet dumping cannot be performed on a pcap_open_dead pcap_t"); + return (-1); +} + +static PAirpcapHandle +pcap_get_airpcap_handle_dead(pcap_t *p) +{ + return (NULL); +} +#endif /* _WIN32 */ -static char *full_pcap_version_string; +static void +pcap_cleanup_dead(pcap_t *p _U_) +{ + /* Nothing to do. */ +} -const char * -pcap_lib_version (void) +pcap_t * +pcap_open_dead_with_tstamp_precision(int linktype, int snaplen, u_int precision) { - char *packet_version_string; - size_t full_pcap_version_string_len; - static char dospfx[] = "DOS-"; + pcap_t *p; + + switch (precision) { + + case PCAP_TSTAMP_PRECISION_MICRO: + case PCAP_TSTAMP_PRECISION_NANO: + break; - if (full_pcap_version_string == NULL) { + default: /* - * Generate the version string. + * This doesn't really matter, but we don't have any way + * to report particular errors, so the only failure we + * should have is a memory allocation failure. Just + * pick microsecond precision. */ - full_pcap_version_string_len = - sizeof dospfx + strlen(pcap_version_string); - full_pcap_version_string = - malloc(full_pcap_version_string_len); - if (full_pcap_version_string == NULL) - return (NULL); - strcpy(full_pcap_version_string, dospfx); - strcat(full_pcap_version_string, pcap_version_string); + precision = PCAP_TSTAMP_PRECISION_MICRO; + break; } - return (full_pcap_version_string); -} + p = malloc(sizeof(*p)); + if (p == NULL) + return NULL; + memset (p, 0, sizeof(*p)); + p->snapshot = snaplen; + p->linktype = linktype; + p->opt.tstamp_precision = precision; + p->can_set_rfmon_op = pcap_can_set_rfmon_dead; + p->read_op = pcap_read_dead; + p->inject_op = pcap_inject_dead; + p->setfilter_op = pcap_setfilter_dead; + p->setdirection_op = pcap_setdirection_dead; + p->set_datalink_op = pcap_set_datalink_dead; + p->getnonblock_op = pcap_getnonblock_dead; + p->setnonblock_op = pcap_setnonblock_dead; + p->stats_op = pcap_stats_dead; +#ifdef _WIN32 + p->stats_ex_op = pcap_stats_ex_dead; + p->setbuff_op = pcap_setbuff_dead; + p->setmode_op = pcap_setmode_dead; + p->setmintocopy_op = pcap_setmintocopy_dead; + p->getevent_op = pcap_getevent_dead; + p->oid_get_request_op = pcap_oid_get_request_dead; + p->oid_set_request_op = pcap_oid_set_request_dead; + p->sendqueue_transmit_op = pcap_sendqueue_transmit_dead; + p->setuserbuffer_op = pcap_setuserbuffer_dead; + p->live_dump_op = pcap_live_dump_dead; + p->live_dump_ended_op = pcap_live_dump_ended_dead; + p->get_airpcap_handle_op = pcap_get_airpcap_handle_dead; +#endif + p->cleanup_op = pcap_cleanup_dead; -#else /* UN*X */ + /* + * A "dead" pcap_t never requires special BPF code generation. + */ + p->bpf_codegen_flags = 0; -const char * -pcap_lib_version(void) + p->activated = 1; + return (p); +} + +pcap_t * +pcap_open_dead(int linktype, int snaplen) { - return (pcap_version_string); + return (pcap_open_dead_with_tstamp_precision(linktype, snaplen, + PCAP_TSTAMP_PRECISION_MICRO)); } -#endif #ifdef YYDEBUG /* @@ -2306,32 +3943,6 @@ PCAP_API void pcap_set_parser_debug(int value); PCAP_API_DEF void pcap_set_parser_debug(int value) { - extern int pcap_debug; - pcap_debug = value; } #endif - -#ifdef BDEBUG -/* - * Set the internal "debug printout" flag for the filter expression optimizer. - * The code to print that stuff is present only if BDEBUG is defined, so - * the flag, and the routine to set it, are defined only if BDEBUG is - * defined. - * - * This is intended for libpcap developers, not for general use. - * If you want to set these in a program, you'll have to declare this - * routine yourself, with the appropriate DLL import attribute on Windows; - * it's not declared in any header file, and won't be declared in any - * header file provided by libpcap. - */ -PCAP_API void pcap_set_optimizer_debug(int value); - -PCAP_API_DEF void -pcap_set_optimizer_debug(int value) -{ - extern int pcap_optimizer_debug; - - pcap_optimizer_debug = value; -} -#endif diff --git a/freebsd/contrib/libpcap/pcap/can_socketcan.h b/freebsd/contrib/libpcap/pcap/can_socketcan.h index 68d2a131..332d9ff5 100644 --- a/freebsd/contrib/libpcap/pcap/can_socketcan.h +++ b/freebsd/contrib/libpcap/pcap/can_socketcan.h @@ -39,16 +39,18 @@ #ifndef lib_pcap_can_socketcan_h #define lib_pcap_can_socketcan_h +#include <pcap/pcap-inttypes.h> + /* * SocketCAN header, as per Documentation/networking/can.txt in the * Linux source. */ typedef struct { - u_int32_t can_id; - u_int8_t payload_length; - u_int8_t pad; - u_int8_t reserved1; - u_int8_t reserved2; + uint32_t can_id; + uint8_t payload_length; + uint8_t pad; + uint8_t reserved1; + uint8_t reserved2; } pcap_can_socketcan_hdr; #endif diff --git a/freebsd/contrib/libpcap/pcap/compiler-tests.h b/freebsd/contrib/libpcap/pcap/compiler-tests.h new file mode 100644 index 00000000..5e17853a --- /dev/null +++ b/freebsd/contrib/libpcap/pcap/compiler-tests.h @@ -0,0 +1,163 @@ +/* -*- Mode: c; tab-width: 8; indent-tabs-mode: 1; c-basic-offset: 8; -*- */ +/* + * Copyright (c) 1993, 1994, 1995, 1996, 1997 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the Computer Systems + * Engineering Group at Lawrence Berkeley Laboratory. + * 4. Neither the name of the University nor of the Laboratory may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef lib_pcap_compiler_tests_h +#define lib_pcap_compiler_tests_h + +/* + * This was introduced by Clang: + * + * http://clang.llvm.org/docs/LanguageExtensions.html#has-attribute + * + * in some version (which version?); it has been picked up by GCC 5.0. + */ +#ifndef __has_attribute + /* + * It's a macro, so you can check whether it's defined to check + * whether it's supported. + * + * If it's not, define it to always return 0, so that we move on to + * the fallback checks. + */ + #define __has_attribute(x) 0 +#endif + +/* + * Note that the C90 spec's "6.8.1 Conditional inclusion" and the + * C99 spec's and C11 spec's "6.10.1 Conditional inclusion" say: + * + * Prior to evaluation, macro invocations in the list of preprocessing + * tokens that will become the controlling constant expression are + * replaced (except for those macro names modified by the defined unary + * operator), just as in normal text. If the token "defined" is + * generated as a result of this replacement process or use of the + * "defined" unary operator does not match one of the two specified + * forms prior to macro replacement, the behavior is undefined. + * + * so you shouldn't use defined() in a #define that's used in #if or + * #elif. Some versions of Clang, for example, will warn about this. + * + * Instead, we check whether the pre-defined macros for particular + * compilers are defined and, if not, define the "is this version XXX + * or a later version of this compiler" macros as 0. + */ + +/* + * Check whether this is GCC major.minor or a later release, or some + * compiler that claims to be "just like GCC" of that version or a + * later release. + */ + +#if ! defined(__GNUC__) +#define PCAP_IS_AT_LEAST_GNUC_VERSION(major, minor) 0 +#else +#define PCAP_IS_AT_LEAST_GNUC_VERSION(major, minor) \ + (__GNUC__ > (major) || \ + (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor))) +#endif + +/* + * Check whether this is Clang major.minor or a later release. + */ + +#if !defined(__clang__) +#define PCAP_IS_AT_LEAST_CLANG_VERSION(major, minor) 0 +#else +#define PCAP_IS_AT_LEAST_CLANG_VERSION(major, minor) \ + (__clang_major__ > (major) || \ + (__clang_major__ == (major) && __clang_minor__ >= (minor))) +#endif + +/* + * Check whether this is Sun C/SunPro C/Oracle Studio major.minor + * or a later release. + * + * The version number in __SUNPRO_C is encoded in hex BCD, with the + * uppermost hex digit being the major version number, the next + * one or two hex digits being the minor version number, and + * the last digit being the patch version. + * + * It represents the *compiler* version, not the product version; + * see + * + * https://sourceforge.net/p/predef/wiki/Compilers/ + * + * for a partial mapping, which we assume continues for later + * 12.x product releases. + */ + +#if ! defined(__SUNPRO_C) +#define PCAP_IS_AT_LEAST_SUNC_VERSION(major,minor) 0 +#else +#define PCAP_SUNPRO_VERSION_TO_BCD(major, minor) \ + (((minor) >= 10) ? \ + (((major) << 12) | (((minor)/10) << 8) | (((minor)%10) << 4)) : \ + (((major) << 8) | ((minor) << 4))) +#define PCAP_IS_AT_LEAST_SUNC_VERSION(major,minor) \ + (__SUNPRO_C >= PCAP_SUNPRO_VERSION_TO_BCD((major), (minor))) +#endif + +/* + * Check whether this is IBM XL C major.minor or a later release. + * + * The version number in __xlC__ has the major version in the + * upper 8 bits and the minor version in the lower 8 bits. + */ + +#if ! defined(__xlC__) +#define PCAP_IS_AT_LEAST_XL_C_VERSION(major,minor) 0 +#else +#define PCAP_IS_AT_LEAST_XL_C_VERSION(major, minor) \ + (__xlC__ >= (((major) << 8) | (minor))) +#endif + +/* + * Check whether this is HP aC++/HP C major.minor or a later release. + * + * The version number in __HP_aCC is encoded in zero-padded decimal BCD, + * with the "A." stripped off, the uppermost two decimal digits being + * the major version number, the next two decimal digits being the minor + * version number, and the last two decimal digits being the patch version. + * (Strip off the A., remove the . between the major and minor version + * number, and add two digits of patch.) + */ + +#if ! defined(__HP_aCC) +#define PCAP_IS_AT_LEAST_HP_C_VERSION(major,minor) 0 +#else +#define PCAP_IS_AT_LEAST_HP_C_VERSION(major,minor) \ + (__HP_aCC >= ((major)*10000 + (minor)*100)) +#endif + +#endif /* lib_pcap_funcattrs_h */ diff --git a/freebsd/contrib/libpcap/pcap/export-defs.h b/freebsd/contrib/libpcap/pcap/export-defs.h deleted file mode 100644 index a2350579..00000000 --- a/freebsd/contrib/libpcap/pcap/export-defs.h +++ /dev/null @@ -1,108 +0,0 @@ -/* -*- Mode: c; tab-width: 8; indent-tabs-mode: 1; c-basic-offset: 8; -*- */ -/* - * Copyright (c) 1993, 1994, 1995, 1996, 1997 - * The Regents of the University of California. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the Computer Systems - * Engineering Group at Lawrence Berkeley Laboratory. - * 4. Neither the name of the University nor of the Laboratory may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifndef lib_pcap_export_defs_h -#define lib_pcap_export_defs_h - -/* - * PCAP_API_DEF must be used when defining *data* exported from - * libpcap. It can be used when defining *functions* exported - * from libpcap, but it doesn't have to be used there. It - * should not be used in declarations in headers. - * - * PCAP_API must be used when *declaring* data or functions - * exported from libpcap; PCAP_API_DEF won't work on all platforms. - */ - -/* - * Check whether this is GCC major.minor or a later release, or some - * compiler that claims to be "just like GCC" of that version or a - * later release. - */ -#define IS_AT_LEAST_GNUC_VERSION(major, minor) \ - (defined(__GNUC__) && \ - (__GNUC__ > (major) || \ - (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))) - -#if defined(_WIN32) - #ifdef BUILDING_PCAP - /* - * We're compiling libpcap, so we should export functions in our - * API. - */ - #define PCAP_API_DEF __declspec(dllexport) - #else - #define PCAP_API_DEF __declspec(dllimport) - #endif -#elif defined(MSDOS) - /* XXX - does this need special treatment? */ - #define PCAP_API_DEF -#else /* UN*X */ - #ifdef BUILDING_PCAP - /* - * We're compiling libpcap, so we should export functions in our API. - * The compiler might be configured not to export functions from a - * shared library by default, so we might have to explicitly mark - * functions as exported. - */ - #if IS_AT_LEAST_GNUC_VERSION(3, 4) - /* - * GCC 3.4 or later, or some compiler asserting compatibility with - * GCC 3.4 or later, so we have __attribute__((visibility()). - */ - #define PCAP_API_DEF __attribute__((visibility("default"))) - #elif defined(__SUNPRO_C) && (__SUNPRO_C >= 0x550) - /* - * Sun C 5.5 or later, so we have __global. - * (Sun C 5.9 and later also have __attribute__((visibility()), - * but there's no reason to prefer it with Sun C.) - */ - #define PCAP_API_DEF __global - #else - /* - * We don't have anything to say. - */ - #define PCAP_API_DEF - #endif - #else - /* - * We're not building libpcap. - */ - #define PCAP_API_DEF - #endif -#endif /* _WIN32/MSDOS/UN*X */ - -#define PCAP_API PCAP_API_DEF extern - -#endif /* lib_pcap_export_defs_h */ diff --git a/freebsd/contrib/libpcap/pcap/funcattrs.h b/freebsd/contrib/libpcap/pcap/funcattrs.h new file mode 100644 index 00000000..a8b1932f --- /dev/null +++ b/freebsd/contrib/libpcap/pcap/funcattrs.h @@ -0,0 +1,261 @@ +/* -*- Mode: c; tab-width: 8; indent-tabs-mode: 1; c-basic-offset: 8; -*- */ +/* + * Copyright (c) 1993, 1994, 1995, 1996, 1997 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the Computer Systems + * Engineering Group at Lawrence Berkeley Laboratory. + * 4. Neither the name of the University nor of the Laboratory may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef lib_pcap_funcattrs_h +#define lib_pcap_funcattrs_h + +#include <pcap/compiler-tests.h> + +/* + * Attributes to apply to functions and their arguments, using various + * compiler-specific extensions. + */ + +/* + * PCAP_API_DEF must be used when defining *data* exported from + * libpcap. It can be used when defining *functions* exported + * from libpcap, but it doesn't have to be used there. It + * should not be used in declarations in headers. + * + * PCAP_API must be used when *declaring* data or functions + * exported from libpcap; PCAP_API_DEF won't work on all platforms. + */ + +#if defined(_WIN32) + /* + * For Windows: + * + * when building libpcap: + * + * if we're building it as a DLL, we have to declare API + * functions with __declspec(dllexport); + * + * if we're building it as a static library, we don't want + * to do so. + * + * when using libpcap: + * + * if we're using the DLL, calls to its functions are a + * little more efficient if they're declared with + * __declspec(dllimport); + * + * if we're not using the dll, we don't want to declare + * them that way. + * + * So: + * + * if pcap_EXPORTS is defined, we define PCAP_API_DEF as + * __declspec(dllexport); + * + * if PCAP_DLL is defined, we define PCAP_API_DEF as + * __declspec(dllimport); + * + * otherwise, we define PCAP_API_DEF as nothing. + */ + #if defined(pcap_EXPORTS) + /* + * We're compiling libpcap as a DLL, so we should export functions + * in our API. + */ + #define PCAP_API_DEF __declspec(dllexport) + #elif defined(PCAP_DLL) + /* + * We're using libpcap as a DLL, so the calls will be a little more + * efficient if we explicitly import the functions. + */ + #define PCAP_API_DEF __declspec(dllimport) + #else + /* + * Either we're building libpcap as a static library, or we're using + * it as a static library, or we don't know for certain that we're + * using it as a dynamic library, so neither import nor export the + * functions explicitly. + */ + #define PCAP_API_DEF + #endif +#elif defined(MSDOS) + /* XXX - does this need special treatment? */ + #define PCAP_API_DEF +#else /* UN*X */ + #ifdef pcap_EXPORTS + /* + * We're compiling libpcap as a (dynamic) shared library, so we should + * export functions in our API. The compiler might be configured not + * to export functions from a shared library by default, so we might + * have to explicitly mark functions as exported. + */ + #if PCAP_IS_AT_LEAST_GNUC_VERSION(3,4) \ + || PCAP_IS_AT_LEAST_XL_C_VERSION(12,0) + /* + * GCC 3.4 or later, or some compiler asserting compatibility with + * GCC 3.4 or later, or XL C 13.0 or later, so we have + * __attribute__((visibility()). + */ + #define PCAP_API_DEF __attribute__((visibility("default"))) + #elif PCAP_IS_AT_LEAST_SUNC_VERSION(5,5) + /* + * Sun C 5.5 or later, so we have __global. + * (Sun C 5.9 and later also have __attribute__((visibility()), + * but there's no reason to prefer it with Sun C.) + */ + #define PCAP_API_DEF __global + #else + /* + * We don't have anything to say. + */ + #define PCAP_API_DEF + #endif + #else + /* + * We're not building libpcap. + */ + #define PCAP_API_DEF + #endif +#endif /* _WIN32/MSDOS/UN*X */ + +#define PCAP_API PCAP_API_DEF extern + +/* + * PCAP_NORETURN, before a function declaration, means "this function + * never returns". (It must go before the function declaration, e.g. + * "extern PCAP_NORETURN func(...)" rather than after the function + * declaration, as the MSVC version has to go before the declaration.) + * + * PCAP_NORETURN_DEF, before a function *definition*, means "this + * function never returns"; it would be used only for static functions + * that are defined before any use, and thus have no declaration. + * (MSVC doesn't support that; I guess the "decl" in "__declspec" + * means "declaration", and __declspec doesn't work with definitions.) + */ +#if __has_attribute(noreturn) \ + || PCAP_IS_AT_LEAST_GNUC_VERSION(2,5) \ + || PCAP_IS_AT_LEAST_SUNC_VERSION(5,9) \ + || PCAP_IS_AT_LEAST_XL_C_VERSION(10,1) \ + || PCAP_IS_AT_LEAST_HP_C_VERSION(6,10) + /* + * Compiler with support for __attribute((noreturn)), or GCC 2.5 and + * later, or Solaris Studio 12 (Sun C 5.9) and later, or IBM XL C 10.1 + * and later (do any earlier versions of XL C support this?), or + * HP aCC A.06.10 and later. + */ + #define PCAP_NORETURN __attribute((noreturn)) + #define PCAP_NORETURN_DEF __attribute((noreturn)) +#elif defined(_MSC_VER) + /* + * MSVC. + */ + #define PCAP_NORETURN __declspec(noreturn) + #define PCAP_NORETURN_DEF +#else + #define PCAP_NORETURN + #define PCAP_NORETURN_DEF +#endif + +/* + * PCAP_PRINTFLIKE(x,y), after a function declaration, means "this function + * does printf-style formatting, with the xth argument being the format + * string and the yth argument being the first argument for the format + * string". + */ +#if __has_attribute(__format__) \ + || PCAP_IS_AT_LEAST_GNUC_VERSION(2,3) \ + || PCAP_IS_AT_LEAST_XL_C_VERSION(10,1) \ + || PCAP_IS_AT_LEAST_HP_C_VERSION(6,10) + /* + * Compiler with support for it, or GCC 2.3 and later, or IBM XL C 10.1 + * and later (do any earlier versions of XL C support this?), + * or HP aCC A.06.10 and later. + */ + #define PCAP_PRINTFLIKE(x,y) __attribute__((__format__(__printf__,x,y))) +#else + #define PCAP_PRINTFLIKE(x,y) +#endif + +/* + * PCAP_DEPRECATED(func, msg), after a function declaration, marks the + * function as deprecated. + * + * The first argument is the name of the function; the second argument is + * a string giving the warning message to use if the compiler supports that. + * + * (Thank you, Microsoft, for requiring the function name.) + */ +#if __has_attribute(deprecated) \ + || PCAP_IS_AT_LEAST_GNUC_VERSION(4,5) \ + || PCAP_IS_AT_LEAST_SUNC_VERSION(5,13) + /* + * Compiler that supports __has_attribute and __attribute__((deprecated)), + * or GCC 4.5 and later, or Sun/Oracle C 12.4 (Sun C 5.13) or later. + * + * Those support __attribute__((deprecated(msg))) (we assume, perhaps + * incorrectly, that anything that supports __has_attribute() is + * recent enough to support __attribute__((deprecated(msg)))). + */ + #define PCAP_DEPRECATED(func, msg) __attribute__((deprecated(msg))) +#elif PCAP_IS_AT_LEAST_GNUC_VERSION(3,1) + /* + * GCC 3.1 through 4.4. + * + * Those support __attribute__((deprecated)) but not + * __attribute__((deprecated(msg))). + */ + #define PCAP_DEPRECATED(func, msg) __attribute__((deprecated)) +#elif (defined(_MSC_VER) && (_MSC_VER >= 1500)) && !defined(BUILDING_PCAP) + /* + * MSVC from Visual Studio 2008 or later, and we're not building + * libpcap itself. + * + * If we *are* building libpcap, we don't want this, as it'll warn + * us even if we *define* the function. + */ + #define PCAP_DEPRECATED(func, msg) __pragma(deprecated(func)) +#else + #define PCAP_DEPRECATED(func, msg) +#endif + +/* + * For flagging arguments as format strings in MSVC. + */ +#ifdef _MSC_VER + #include <sal.h> + #if _MSC_VER > 1400 + #define PCAP_FORMAT_STRING(p) _Printf_format_string_ p + #else + #define PCAP_FORMAT_STRING(p) __format_string p + #endif +#else + #define PCAP_FORMAT_STRING(p) p +#endif + +#endif /* lib_pcap_funcattrs_h */ diff --git a/freebsd/contrib/libpcap/pcap/namedb.h b/freebsd/contrib/libpcap/pcap/namedb.h index 73fb40a4..c66846d3 100644 --- a/freebsd/contrib/libpcap/pcap/namedb.h +++ b/freebsd/contrib/libpcap/pcap/namedb.h @@ -60,9 +60,7 @@ PCAP_API u_char *pcap_ether_hostton(const char*); PCAP_API u_char *pcap_ether_aton(const char *); PCAP_API bpf_u_int32 **pcap_nametoaddr(const char *); -#ifdef INET6 PCAP_API struct addrinfo *pcap_nametoaddrinfo(const char *); -#endif PCAP_API bpf_u_int32 pcap_nametonetaddr(const char *); PCAP_API int pcap_nametoport(const char *, int *, int *); diff --git a/freebsd/contrib/libpcap/pcap/nflog.h b/freebsd/contrib/libpcap/pcap/nflog.h index a3867cdd..29a71d2d 100644 --- a/freebsd/contrib/libpcap/pcap/nflog.h +++ b/freebsd/contrib/libpcap/pcap/nflog.h @@ -28,6 +28,8 @@ #ifndef lib_pcap_nflog_h #define lib_pcap_nflog_h +#include <pcap/pcap-inttypes.h> + /* * Structure of an NFLOG header and TLV parts, as described at * http://www.tcpdump.org/linktypes/LINKTYPE_NFLOG.html @@ -40,32 +42,32 @@ * data, etc.). */ typedef struct nflog_hdr { - u_int8_t nflog_family; /* address family */ - u_int8_t nflog_version; /* version */ - u_int16_t nflog_rid; /* resource ID */ + uint8_t nflog_family; /* address family */ + uint8_t nflog_version; /* version */ + uint16_t nflog_rid; /* resource ID */ } nflog_hdr_t; typedef struct nflog_tlv { - u_int16_t tlv_length; /* tlv length */ - u_int16_t tlv_type; /* tlv type */ + uint16_t tlv_length; /* tlv length */ + uint16_t tlv_type; /* tlv type */ /* value follows this */ } nflog_tlv_t; typedef struct nflog_packet_hdr { - u_int16_t hw_protocol; /* hw protocol */ - u_int8_t hook; /* netfilter hook */ - u_int8_t pad; /* padding to 32 bits */ + uint16_t hw_protocol; /* hw protocol */ + uint8_t hook; /* netfilter hook */ + uint8_t pad; /* padding to 32 bits */ } nflog_packet_hdr_t; typedef struct nflog_hwaddr { - u_int16_t hw_addrlen; /* address length */ - u_int16_t pad; /* padding to 32-bit boundary */ - u_int8_t hw_addr[8]; /* address, up to 8 bytes */ + uint16_t hw_addrlen; /* address length */ + uint16_t pad; /* padding to 32-bit boundary */ + uint8_t hw_addr[8]; /* address, up to 8 bytes */ } nflog_hwaddr_t; typedef struct nflog_timestamp { - u_int64_t sec; - u_int64_t usec; + uint64_t sec; + uint64_t usec; } nflog_timestamp_t; /* diff --git a/freebsd/contrib/libpcap/pcap/pcap-inttypes.h b/freebsd/contrib/libpcap/pcap/pcap-inttypes.h new file mode 100644 index 00000000..af2c23c8 --- /dev/null +++ b/freebsd/contrib/libpcap/pcap/pcap-inttypes.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2002 - 2005 NetGroup, Politecnico di Torino (Italy) + * Copyright (c) 2005 - 2009 CACE Technologies, Inc. Davis (California) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the Politecnico di Torino nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef pcap_pcap_inttypes_h +#define pcap_pcap_inttypes_h + +/* + * Get the integer types and PRi[doux]64 values from C99 <inttypes.h> + * defined, by hook or by crook. + */ +#if defined(_MSC_VER) + /* + * Compiler is MSVC. + */ + #if _MSC_VER >= 1800 + /* + * VS 2013 or newer; we have <inttypes.h>. + */ + #include <inttypes.h> + #else + /* + * Earlier VS; we have to define this stuff ourselves. + */ + typedef unsigned char uint8_t; + typedef signed char int8_t; + typedef unsigned short uint16_t; + typedef signed short int16_t; + typedef unsigned int uint32_t; + typedef signed int int32_t; + #ifdef _MSC_EXTENSIONS + typedef unsigned _int64 uint64_t; + typedef _int64 int64_t; + #else /* _MSC_EXTENSIONS */ + typedef unsigned long long uint64_t; + typedef long long int64_t; + #endif + #endif + + /* + * These may be defined by <inttypes.h>. + * + * XXX - for MSVC, we always want the _MSC_EXTENSIONS versions. + * What about other compilers? If, as the MinGW Web site says MinGW + * does, the other compilers just use Microsoft's run-time library, + * then they should probably use the _MSC_EXTENSIONS even if the + * compiler doesn't define _MSC_EXTENSIONS. + * + * XXX - we currently aren't using any of these, but this allows + * their use in the future. + */ + #ifndef PRId64 + #ifdef _MSC_EXTENSIONS + #define PRId64 "I64d" + #else + #define PRId64 "lld" + #endif + #endif /* PRId64 */ + + #ifndef PRIo64 + #ifdef _MSC_EXTENSIONS + #define PRIo64 "I64o" + #else + #define PRIo64 "llo" + #endif + #endif /* PRIo64 */ + + #ifndef PRIx64 + #ifdef _MSC_EXTENSIONS + #define PRIx64 "I64x" + #else + #define PRIx64 "llx" + #endif + #endif + + #ifndef PRIu64 + #ifdef _MSC_EXTENSIONS + #define PRIu64 "I64u" + #else + #define PRIu64 "llu" + #endif + #endif +#elif defined(__MINGW32__) || !defined(_WIN32) + /* + * Compiler is MinGW or target is UN*X or MS-DOS. Just use + * <inttypes.h>. + */ + #include <inttypes.h> +#endif + +#endif /* pcap/pcap-inttypes.h */ diff --git a/freebsd/contrib/libpcap/pcap/pcap.h b/freebsd/contrib/libpcap/pcap/pcap.h index 9c277124..dbe70dfd 100644 --- a/freebsd/contrib/libpcap/pcap/pcap.h +++ b/freebsd/contrib/libpcap/pcap/pcap.h @@ -32,18 +32,55 @@ * SUCH DAMAGE. */ +/* + * Remote packet capture mechanisms and extensions from WinPcap: + * + * Copyright (c) 2002 - 2003 + * NetGroup, Politecnico di Torino (Italy) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the Politecnico di Torino nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + #ifndef lib_pcap_pcap_h #define lib_pcap_pcap_h -#include <pcap/export-defs.h> +#include <pcap/funcattrs.h> + +#include <pcap/pcap-inttypes.h> #if defined(_WIN32) - #include <pcap-stdinc.h> + #include <winsock2.h> /* u_int, u_char etc. */ + #include <io.h> /* _get_osfhandle() */ #elif defined(MSDOS) - #include <sys/types.h> - #include <sys/socket.h> /* u_int, u_char etc. */ + #include <sys/types.h> /* u_int, u_char etc. */ + #include <sys/socket.h> #else /* UN*X */ - #include <sys/types.h> + #include <sys/types.h> /* u_int, u_char etc. */ #include <sys/time.h> #endif /* _WIN32/MSDOS/UN*X */ @@ -170,11 +207,11 @@ struct pcap_stat { u_int ps_recv; /* number of packets received */ u_int ps_drop; /* number of packets dropped */ u_int ps_ifdrop; /* drops by interface -- only supported on some platforms */ -#if defined(_WIN32) && defined(HAVE_REMOTE) +#ifdef _WIN32 u_int ps_capt; /* number of packets that reach the application */ u_int ps_sent; /* number of packets sent by the server on the network */ u_int ps_netdrop; /* number of packets lost on the network */ -#endif /* _WIN32 && HAVE_REMOTE */ +#endif /* _WIN32 */ }; #ifdef MSDOS @@ -221,9 +258,15 @@ struct pcap_if { bpf_u_int32 flags; /* PCAP_IF_ interface flags */ }; -#define PCAP_IF_LOOPBACK 0x00000001 /* interface is loopback */ -#define PCAP_IF_UP 0x00000002 /* interface is up */ -#define PCAP_IF_RUNNING 0x00000004 /* interface is running */ +#define PCAP_IF_LOOPBACK 0x00000001 /* interface is loopback */ +#define PCAP_IF_UP 0x00000002 /* interface is up */ +#define PCAP_IF_RUNNING 0x00000004 /* interface is running */ +#define PCAP_IF_WIRELESS 0x00000008 /* interface is wireless (*NOT* necessarily Wi-Fi!) */ +#define PCAP_IF_CONNECTION_STATUS 0x00000030 /* connection status: */ +#define PCAP_IF_CONNECTION_STATUS_UNKNOWN 0x00000000 /* unknown */ +#define PCAP_IF_CONNECTION_STATUS_CONNECTED 0x00000010 /* connected */ +#define PCAP_IF_CONNECTION_STATUS_DISCONNECTED 0x00000020 /* disconnected */ +#define PCAP_IF_CONNECTION_STATUS_NOT_APPLICABLE 0x00000030 /* not applicable */ /* * Representation of an interface address. @@ -273,7 +316,14 @@ typedef void (*pcap_handler)(u_char *, const struct pcap_pkthdr *, */ #define PCAP_NETMASK_UNKNOWN 0xffffffff -PCAP_API char *pcap_lookupdev(char *); +/* + * We're deprecating pcap_lookupdev() for various reasons (not + * thread-safe, can behave weirdly with WinPcap). Callers + * should use pcap_findalldevs() and use the first device. + */ +PCAP_API char *pcap_lookupdev(char *) +PCAP_DEPRECATED(pcap_lookupdev, "use 'pcap_findalldevs' and use the first device"); + PCAP_API int pcap_lookupnet(const char *, bpf_u_int32 *, bpf_u_int32 *, char *); PCAP_API pcap_t *pcap_create(const char *, char *); @@ -295,6 +345,10 @@ PCAP_API int pcap_tstamp_type_name_to_val(const char *); PCAP_API const char *pcap_tstamp_type_val_to_name(int); PCAP_API const char *pcap_tstamp_type_val_to_description(int); +#ifdef __linux__ +PCAP_API int pcap_set_protocol(pcap_t *, int); +#endif + /* * Time stamp types. * Not all systems and interfaces will necessarily support all of these. @@ -358,7 +412,16 @@ PCAP_API pcap_t *pcap_open_offline(const char *, char *); PCAP_API pcap_t *pcap_hopen_offline(intptr_t, char *); /* * If we're building libpcap, these are internal routines in savefile.c, - * so we mustn't define them as macros. + * so we must not define them as macros. + * + * If we're not building libpcap, given that the version of the C runtime + * with which libpcap was built might be different from the version + * of the C runtime with which an application using libpcap was built, + * and that a FILE structure may differ between the two versions of the + * C runtime, calls to _fileno() must use the version of _fileno() in + * the C runtime used to open the FILE *, not the version in the C + * runtime with which libpcap was built. (Maybe once the Universal CRT + * rules the world, this will cease to be a problem.) */ #ifndef BUILDING_PCAP #define pcap_fopen_offline_with_tstamp_precision(f,p,b) \ @@ -407,6 +470,7 @@ PCAP_API int pcap_snapshot(pcap_t *); PCAP_API int pcap_is_swapped(pcap_t *); PCAP_API int pcap_major_version(pcap_t *); PCAP_API int pcap_minor_version(pcap_t *); +PCAP_API int pcap_bufsize(pcap_t *); /* XXX */ PCAP_API FILE *pcap_file(pcap_t *); @@ -421,6 +485,7 @@ PCAP_API pcap_dumper_t *pcap_dump_fopen(pcap_t *, FILE *fp); PCAP_API pcap_dumper_t *pcap_dump_open_append(pcap_t *, const char *); PCAP_API FILE *pcap_dump_file(pcap_dumper_t *); PCAP_API long pcap_dump_ftell(pcap_dumper_t *); +PCAP_API int64_t pcap_dump_ftell64(pcap_dumper_t *); PCAP_API int pcap_dump_flush(pcap_dumper_t *); PCAP_API void pcap_dump_close(pcap_dumper_t *); PCAP_API void pcap_dump(u_char *, const struct pcap_pkthdr *, const u_char *); @@ -428,6 +493,21 @@ PCAP_API void pcap_dump(u_char *, const struct pcap_pkthdr *, const u_char *); PCAP_API int pcap_findalldevs(pcap_if_t **, char *); PCAP_API void pcap_freealldevs(pcap_if_t *); +/* + * We return a pointer to the version string, rather than exporting the + * version string directly. + * + * On at least some UNIXes, if you import data from a shared library into + * an program, the data is bound into the program binary, so if the string + * in the version of the library with which the program was linked isn't + * the same as the string in the version of the library with which the + * program is being run, various undesirable things may happen (warnings, + * the string being the one from the version of the library with which the + * program was linked, or even weirder things, such as the string being the + * one from the library but being truncated). + * + * On Windows, the string is constructed at run time. + */ PCAP_API const char *pcap_lib_version(void); /* @@ -455,7 +535,7 @@ PCAP_API void bpf_dump(const struct bpf_program *, int); */ struct pcap_send_queue { - u_int maxlen; /* Maximum size of the the queue, in bytes. This + u_int maxlen; /* Maximum size of the queue, in bytes. This variable contains the size of the buffer field. */ u_int len; /* Current size of the queue, in bytes. */ char *buffer; /* Buffer containing the packets to be sent. */ @@ -521,13 +601,370 @@ PCAP_API void bpf_dump(const struct bpf_program *, int); */ PCAP_API int pcap_get_selectable_fd(pcap_t *); + PCAP_API struct timeval *pcap_get_required_select_timeout(pcap_t *); #endif /* _WIN32/MSDOS/UN*X */ -#ifdef HAVE_REMOTE - /* Includes most of the public stuff that is needed for the remote capture */ - #include <remote-ext.h> -#endif /* HAVE_REMOTE */ +#if 0 /* Remote capture is disabled on FreeBSD */ +/* + * Remote capture definitions. + * + * These routines are only present if libpcap has been configured to + * include remote capture support. + */ + +/* + * The maximum buffer size in which address, port, interface names are kept. + * + * In case the adapter name or such is larger than this value, it is truncated. + * This is not used by the user; however it must be aware that an hostname / interface + * name longer than this value will be truncated. + */ +#define PCAP_BUF_SIZE 1024 + +/* + * The type of input source, passed to pcap_open(). + */ +#define PCAP_SRC_FILE 2 /* local savefile */ +#define PCAP_SRC_IFLOCAL 3 /* local network interface */ +#define PCAP_SRC_IFREMOTE 4 /* interface on a remote host, using RPCAP */ + +/* + * The formats allowed by pcap_open() are the following: + * - file://path_and_filename [opens a local file] + * - rpcap://devicename [opens the selected device devices available on the local host, without using the RPCAP protocol] + * - rpcap://host/devicename [opens the selected device available on a remote host] + * - rpcap://host:port/devicename [opens the selected device available on a remote host, using a non-standard port for RPCAP] + * - adaptername [to open a local adapter; kept for compability, but it is strongly discouraged] + * - (NULL) [to open the first local adapter; kept for compability, but it is strongly discouraged] + * + * The formats allowed by the pcap_findalldevs_ex() are the following: + * - file://folder/ [lists all the files in the given folder] + * - rpcap:// [lists all local adapters] + * - rpcap://host:port/ [lists the devices available on a remote host] + * + * Referring to the 'host' and 'port' parameters, they can be either numeric or literal. Since + * IPv6 is fully supported, these are the allowed formats: + * + * - host (literal): e.g. host.foo.bar + * - host (numeric IPv4): e.g. 10.11.12.13 + * - host (numeric IPv4, IPv6 style): e.g. [10.11.12.13] + * - host (numeric IPv6): e.g. [1:2:3::4] + * - port: can be either numeric (e.g. '80') or literal (e.g. 'http') + * + * Here you find some allowed examples: + * - rpcap://host.foo.bar/devicename [everything literal, no port number] + * - rpcap://host.foo.bar:1234/devicename [everything literal, with port number] + * - rpcap://10.11.12.13/devicename [IPv4 numeric, no port number] + * - rpcap://10.11.12.13:1234/devicename [IPv4 numeric, with port number] + * - rpcap://[10.11.12.13]:1234/devicename [IPv4 numeric with IPv6 format, with port number] + * - rpcap://[1:2:3::4]/devicename [IPv6 numeric, no port number] + * - rpcap://[1:2:3::4]:1234/devicename [IPv6 numeric, with port number] + * - rpcap://[1:2:3::4]:http/devicename [IPv6 numeric, with literal port number] + */ + +/* + * URL schemes for capture source. + */ +/* + * This string indicates that the user wants to open a capture from a + * local file. + */ +#define PCAP_SRC_FILE_STRING "file://" +/* + * This string indicates that the user wants to open a capture from a + * network interface. This string does not necessarily involve the use + * of the RPCAP protocol. If the interface required resides on the local + * host, the RPCAP protocol is not involved and the local functions are used. + */ +#define PCAP_SRC_IF_STRING "rpcap://" + +/* + * Flags to pass to pcap_open(). + */ + +/* + * Specifies whether promiscuous mode is to be used. + */ +#define PCAP_OPENFLAG_PROMISCUOUS 0x00000001 + +/* + * Specifies, for an RPCAP capture, whether the data transfer (in + * case of a remote capture) has to be done with UDP protocol. + * + * If it is '1' if you want a UDP data connection, '0' if you want + * a TCP data connection; control connection is always TCP-based. + * A UDP connection is much lighter, but it does not guarantee that all + * the captured packets arrive to the client workstation. Moreover, + * it could be harmful in case of network congestion. + * This flag is meaningless if the source is not a remote interface. + * In that case, it is simply ignored. + */ +#define PCAP_OPENFLAG_DATATX_UDP 0x00000002 + +/* + * Specifies wheether the remote probe will capture its own generated + * traffic. + * + * In case the remote probe uses the same interface to capture traffic + * and to send data back to the caller, the captured traffic includes + * the RPCAP traffic as well. If this flag is turned on, the RPCAP + * traffic is excluded from the capture, so that the trace returned + * back to the collector is does not include this traffic. + * + * Has no effect on local interfaces or savefiles. + */ +#define PCAP_OPENFLAG_NOCAPTURE_RPCAP 0x00000004 + +/* + * Specifies whether the local adapter will capture its own generated traffic. + * + * This flag tells the underlying capture driver to drop the packets + * that were sent by itself. This is useful when building applications + * such as bridges that should ignore the traffic they just sent. + * + * Supported only on Windows. + */ +#define PCAP_OPENFLAG_NOCAPTURE_LOCAL 0x00000008 + +/* + * This flag configures the adapter for maximum responsiveness. + * + * In presence of a large value for nbytes, WinPcap waits for the arrival + * of several packets before copying the data to the user. This guarantees + * a low number of system calls, i.e. lower processor usage, i.e. better + * performance, which is good for applications like sniffers. If the user + * sets the PCAP_OPENFLAG_MAX_RESPONSIVENESS flag, the capture driver will + * copy the packets as soon as the application is ready to receive them. + * This is suggested for real time applications (such as, for example, + * a bridge) that need the best responsiveness. + * + * The equivalent with pcap_create()/pcap_activate() is "immediate mode". + */ +#define PCAP_OPENFLAG_MAX_RESPONSIVENESS 0x00000010 + +/* + * Remote authentication methods. + * These are used in the 'type' member of the pcap_rmtauth structure. + */ + +/* + * NULL authentication. + * + * The 'NULL' authentication has to be equal to 'zero', so that old + * applications can just put every field of struct pcap_rmtauth to zero, + * and it does work. + */ +#define RPCAP_RMTAUTH_NULL 0 +/* + * Username/password authentication. + * + * With this type of authentication, the RPCAP protocol will use the username/ + * password provided to authenticate the user on the remote machine. If the + * authentication is successful (and the user has the right to open network + * devices) the RPCAP connection will continue; otherwise it will be dropped. + * + * *******NOTE********: the username and password are sent over the network + * to the capture server *IN CLEAR TEXT*. Don't use this on a network + * that you don't completely control! (And be *really* careful in your + * definition of "completely"!) + */ +#define RPCAP_RMTAUTH_PWD 1 + +/* + * This structure keeps the information needed to autheticate the user + * on a remote machine. + * + * The remote machine can either grant or refuse the access according + * to the information provided. + * In case the NULL authentication is required, both 'username' and + * 'password' can be NULL pointers. + * + * This structure is meaningless if the source is not a remote interface; + * in that case, the functions which requires such a structure can accept + * a NULL pointer as well. + */ +struct pcap_rmtauth +{ + /* + * \brief Type of the authentication required. + * + * In order to provide maximum flexibility, we can support different types + * of authentication based on the value of this 'type' variable. The currently + * supported authentication methods are defined into the + * \link remote_auth_methods Remote Authentication Methods Section\endlink. + */ + int type; + /* + * \brief Zero-terminated string containing the username that has to be + * used on the remote machine for authentication. + * + * This field is meaningless in case of the RPCAP_RMTAUTH_NULL authentication + * and it can be NULL. + */ + char *username; + /* + * \brief Zero-terminated string containing the password that has to be + * used on the remote machine for authentication. + * + * This field is meaningless in case of the RPCAP_RMTAUTH_NULL authentication + * and it can be NULL. + */ + char *password; +}; + +/* + * This routine can open a savefile, a local device, or a device on + * a remote machine running an RPCAP server. + * + * For opening a savefile, the pcap_open_offline routines can be used, + * and will work just as well; code using them will work on more + * platforms than code using pcap_open() to open savefiles. + * + * For opening a local device, pcap_open_live() can be used; it supports + * most of the capabilities that pcap_open() supports, and code using it + * will work on more platforms than code using pcap_open(). pcap_create() + * and pcap_activate() can also be used; they support all capabilities + * that pcap_open() supports, except for the Windows-only + * PCAP_OPENFLAG_NOCAPTURE_LOCAL, and support additional capabilities. + * + * For opening a remote capture, pcap_open() is currently the only + * API available. + */ +PCAP_API pcap_t *pcap_open(const char *source, int snaplen, int flags, + int read_timeout, struct pcap_rmtauth *auth, char *errbuf); +PCAP_API int pcap_createsrcstr(char *source, int type, const char *host, + const char *port, const char *name, char *errbuf); +PCAP_API int pcap_parsesrcstr(const char *source, int *type, char *host, + char *port, char *name, char *errbuf); + +/* + * This routine can scan a directory for savefiles, list local capture + * devices, or list capture devices on a remote machine running an RPCAP + * server. + * + * For scanning for savefiles, it can be used on both UN*X systems and + * Windows systems; for each directory entry it sees, it tries to open + * the file as a savefile using pcap_open_offline(), and only includes + * it in the list of files if the open succeeds, so it filters out + * files for which the user doesn't have read permission, as well as + * files that aren't valid savefiles readable by libpcap. + * + * For listing local capture devices, it's just a wrapper around + * pcap_findalldevs(); code using pcap_findalldevs() will work on more + * platforms than code using pcap_findalldevs_ex(). + * + * For listing remote capture devices, pcap_findalldevs_ex() is currently + * the only API available. + */ +PCAP_API int pcap_findalldevs_ex(char *source, struct pcap_rmtauth *auth, + pcap_if_t **alldevs, char *errbuf); + +/* + * Sampling methods. + * + * These allow pcap_loop(), pcap_dispatch(), pcap_next(), and pcap_next_ex() + * to see only a sample of packets, rather than all packets. + * + * Currently, they work only on Windows local captures. + */ + +/* + * Specifies that no sampling is to be done on the current capture. + * + * In this case, no sampling algorithms are applied to the current capture. + */ +#define PCAP_SAMP_NOSAMP 0 + +/* + * Specifies that only 1 out of N packets must be returned to the user. + * + * In this case, the 'value' field of the 'pcap_samp' structure indicates the + * number of packets (minus 1) that must be discarded before one packet got + * accepted. + * In other words, if 'value = 10', the first packet is returned to the + * caller, while the following 9 are discarded. + */ +#define PCAP_SAMP_1_EVERY_N 1 + +/* + * Specifies that we have to return 1 packet every N milliseconds. + * + * In this case, the 'value' field of the 'pcap_samp' structure indicates + * the 'waiting time' in milliseconds before one packet got accepted. + * In other words, if 'value = 10', the first packet is returned to the + * caller; the next returned one will be the first packet that arrives + * when 10ms have elapsed. + */ +#define PCAP_SAMP_FIRST_AFTER_N_MS 2 + +/* + * This structure defines the information related to sampling. + * + * In case the sampling is requested, the capturing device should read + * only a subset of the packets coming from the source. The returned packets + * depend on the sampling parameters. + * + * WARNING: The sampling process is applied *after* the filtering process. + * In other words, packets are filtered first, then the sampling process + * selects a subset of the 'filtered' packets and it returns them to the + * caller. + */ +struct pcap_samp +{ + /* + * Method used for sampling; see above. + */ + int method; + + /* + * This value depends on the sampling method defined. + * For its meaning, see above. + */ + int value; +}; + +/* + * New functions. + */ +PCAP_API struct pcap_samp *pcap_setsampling(pcap_t *p); + +/* + * RPCAP active mode. + */ + +/* Maximum length of an host name (needed for the RPCAP active mode) */ +#define RPCAP_HOSTLIST_SIZE 1024 + +/* + * Some minor differences between UN*X sockets and and Winsock sockets. + */ +#ifndef _WIN32 + /*! + * \brief In Winsock, a socket handle is of type SOCKET; in UN*X, it's + * a file descriptor, and therefore a signed integer. + * We define SOCKET to be a signed integer on UN*X, so that it can + * be used on both platforms. + */ + #define SOCKET int + + /*! + * \brief In Winsock, the error return if socket() fails is INVALID_SOCKET; + * in UN*X, it's -1. + * We define INVALID_SOCKET to be -1 on UN*X, so that it can be used on + * both platforms. + */ + #define INVALID_SOCKET -1 +#endif + +PCAP_API SOCKET pcap_remoteact_accept(const char *address, const char *port, + const char *hostlist, char *connectinghost, + struct pcap_rmtauth *auth, char *errbuf); +PCAP_API int pcap_remoteact_list(char *hostlist, char sep, int size, + char *errbuf); +PCAP_API int pcap_remoteact_close(const char *host, char *errbuf); +PCAP_API void pcap_remoteact_cleanup(void); +#endif /* Remote capture is disabled on FreeBSD */ #ifdef __cplusplus } diff --git a/freebsd/contrib/libpcap/pcap/sll.h b/freebsd/contrib/libpcap/pcap/sll.h index b46d15f2..c4d08862 100644 --- a/freebsd/contrib/libpcap/pcap/sll.h +++ b/freebsd/contrib/libpcap/pcap/sll.h @@ -80,12 +80,14 @@ #define SLL_HDR_LEN 16 /* total header length */ #define SLL_ADDRLEN 8 /* length of address field */ +#include <pcap/pcap-inttypes.h> + struct sll_header { - u_int16_t sll_pkttype; /* packet type */ - u_int16_t sll_hatype; /* link-layer address type */ - u_int16_t sll_halen; /* link-layer address length */ - u_int8_t sll_addr[SLL_ADDRLEN]; /* link-layer address */ - u_int16_t sll_protocol; /* protocol */ + uint16_t sll_pkttype; /* packet type */ + uint16_t sll_hatype; /* link-layer address type */ + uint16_t sll_halen; /* link-layer address length */ + uint8_t sll_addr[SLL_ADDRLEN]; /* link-layer address */ + uint16_t sll_protocol; /* protocol */ }; /* diff --git a/freebsd/contrib/libpcap/pcap/usb.h b/freebsd/contrib/libpcap/pcap/usb.h index 26a9046b..e485ec85 100644 --- a/freebsd/contrib/libpcap/pcap/usb.h +++ b/freebsd/contrib/libpcap/pcap/usb.h @@ -34,6 +34,8 @@ #ifndef lib_pcap_usb_h #define lib_pcap_usb_h +#include <pcap/pcap-inttypes.h> + /* * possible transfer mode */ @@ -55,11 +57,11 @@ * Appears at the front of each Control S-type packet in DLT_USB captures. */ typedef struct _usb_setup { - u_int8_t bmRequestType; - u_int8_t bRequest; - u_int16_t wValue; - u_int16_t wIndex; - u_int16_t wLength; + uint8_t bmRequestType; + uint8_t bRequest; + uint16_t wValue; + uint16_t wIndex; + uint16_t wLength; } pcap_usb_setup; /* @@ -75,19 +77,19 @@ typedef struct _iso_rec { * Appears at the front of each packet in DLT_USB_LINUX captures. */ typedef struct _usb_header { - u_int64_t id; - u_int8_t event_type; - u_int8_t transfer_type; - u_int8_t endpoint_number; - u_int8_t device_address; - u_int16_t bus_id; + uint64_t id; + uint8_t event_type; + uint8_t transfer_type; + uint8_t endpoint_number; + uint8_t device_address; + uint16_t bus_id; char setup_flag;/*if !=0 the urb setup header is not present*/ char data_flag; /*if !=0 no urb data is present*/ int64_t ts_sec; int32_t ts_usec; int32_t status; - u_int32_t urb_len; - u_int32_t data_len; /* amount of urb data really present in this event*/ + uint32_t urb_len; + uint32_t data_len; /* amount of urb data really present in this event*/ pcap_usb_setup setup; } pcap_usb_header; @@ -100,27 +102,27 @@ typedef struct _usb_header { * Appears at the front of each packet in DLT_USB_LINUX_MMAPPED captures. */ typedef struct _usb_header_mmapped { - u_int64_t id; - u_int8_t event_type; - u_int8_t transfer_type; - u_int8_t endpoint_number; - u_int8_t device_address; - u_int16_t bus_id; + uint64_t id; + uint8_t event_type; + uint8_t transfer_type; + uint8_t endpoint_number; + uint8_t device_address; + uint16_t bus_id; char setup_flag;/*if !=0 the urb setup header is not present*/ char data_flag; /*if !=0 no urb data is present*/ int64_t ts_sec; int32_t ts_usec; int32_t status; - u_int32_t urb_len; - u_int32_t data_len; /* amount of urb data really present in this event*/ + uint32_t urb_len; + uint32_t data_len; /* amount of urb data really present in this event*/ union { pcap_usb_setup setup; iso_rec iso; } s; int32_t interval; /* for Interrupt and Isochronous events */ int32_t start_frame; /* for Isochronous events */ - u_int32_t xfer_flags; /* copy of URB's transfer flags */ - u_int32_t ndesc; /* number of isochronous descriptors */ + uint32_t xfer_flags; /* copy of URB's transfer flags */ + uint32_t ndesc; /* number of isochronous descriptors */ } pcap_usb_header_mmapped; /* @@ -133,9 +135,9 @@ typedef struct _usb_header_mmapped { */ typedef struct _usb_isodesc { int32_t status; - u_int32_t offset; - u_int32_t len; - u_int8_t pad[4]; + uint32_t offset; + uint32_t len; + uint8_t pad[4]; } usb_isodesc; #endif diff --git a/freebsd/contrib/libpcap/portability.h b/freebsd/contrib/libpcap/portability.h index 8a6bf40e..b3612542 100644 --- a/freebsd/contrib/libpcap/portability.h +++ b/freebsd/contrib/libpcap/portability.h @@ -39,6 +39,8 @@ * flavors of UN*X. */ +#include "pcap/funcattrs.h" + #ifdef __cplusplus extern "C" { #endif @@ -47,7 +49,7 @@ extern "C" { /* * Macro that does the same thing as strlcpy(). */ - #ifdef _MSC_VER + #if defined(_MSC_VER) || defined(__MINGW32__) /* * strncpy_s() is supported at least back to Visual * Studio 2005. @@ -63,44 +65,37 @@ extern "C" { #endif #endif -/* - * For flagging arguments as format strings in MSVC. - */ -#if _MSC_VER >= 1400 - #include <sal.h> - #if _MSC_VER > 1400 - #define FORMAT_STRING(p) _Printf_format_string_ p - #else - #define FORMAT_STRING(p) __format_string p - #endif -#else - #define FORMAT_STRING(p) p -#endif - -#ifdef _MSC_VER - #define strdup _strdup - #define sscanf sscanf_s - #define setbuf(x, y) \ - setvbuf((x), (y), _IONBF, 0) - #define fopen(x, y) \ - fopen_safe((x), (y)) - FILE *fopen_safe(const char *filename, const char* mode); -#endif - -#if defined(_MSC_VER) || defined(__MINGW32__) +#ifndef HAVE_STRLCAT + /* + * Macro that does the same thing as strlcat(). + */ + #if defined(_MSC_VER) || defined(__MINGW32__) + /* + * strncat_s() is supported at least back to Visual + * Studio 2005. + */ #define strlcat(x, y, z) \ strncat_s((x), (z), (y), _TRUNCATE) + #else + /* + * ANSI C says strncat() always null-terminates its first argument, + * so 1) we don't need to explicitly null-terminate the string + * ourselves and 2) we need to leave room for the null terminator. + */ + #define strlcat(x, y, z) \ + strncat((x), (y), (z) - strlen((x)) - 1) + #endif #endif #ifdef _MSC_VER /* - * MSVC. + * If <crtdbg.h> has been included, and _DEBUG is defined, and + * __STDC__ is zero, <crtdbg.h> will define strdup() to call + * _strdup_dbg(). So if it's already defined, don't redefine + * it. */ - #if _MSC_VER >= 1900 - /* - * VS 2015 or newer; we have snprintf() function. - */ - #define HAVE_SNPRINTF + #ifndef strdup + #define strdup _strdup #endif #endif @@ -129,11 +124,8 @@ extern "C" { #ifdef HAVE_SNPRINTF #define pcap_snprintf snprintf #else -extern int pcap_snprintf(char *, size_t, FORMAT_STRING(const char *), ...) -#ifdef __ATTRIBUTE___FORMAT_OK - __attribute__((format (printf, 3, 4))) -#endif /* __ATTRIBUTE___FORMAT_OK */ - ; +extern int pcap_snprintf(char *, size_t, PCAP_FORMAT_STRING(const char *), ...) + PCAP_PRINTFLIKE(3, 4); #endif #ifdef HAVE_VSNPRINTF @@ -145,7 +137,7 @@ extern int pcap_vsnprintf(char *, size_t, const char *, va_list ap); #ifdef HAVE_STRTOK_R #define pcap_strtok_r strtok_r #else - #ifdef _MSC_VER + #ifdef _WIN32 /* * Microsoft gives it a different name. */ @@ -155,55 +147,11 @@ extern int pcap_vsnprintf(char *, size_t, const char *, va_list ap); * Define it ourselves. */ #define NEED_STRTOK_R - extern int pcap_strtok_r(char *, const char *, char **); + extern char *pcap_strtok_r(char *, const char *, char **); #endif #endif /* HAVE_STRTOK_R */ #ifdef _WIN32 - /* - * These may be defined by <inttypes.h>. - * - * XXX - for MSVC, we always want the _MSC_EXTENSIONS versions. - * What about other compilers? If, as the MinGW Web site says MinGW - * does, the other compilers just use Microsoft's run-time library, - * then they should probably use the _MSC_EXTENSIONS even if the - * compiler doesn't define _MSC_EXTENSIONS. - * - * XXX - we currently aren't using any of these, but this allows - * their use in the future. - */ - #ifndef PRId64 - #ifdef _MSC_EXTENSIONS - #define PRId64 "I64d" - #else - #define PRId64 "lld" - #endif - #endif /* PRId64 */ - - #ifndef PRIo64 - #ifdef _MSC_EXTENSIONS - #define PRIo64 "I64o" - #else - #define PRIo64 "llo" - #endif - #endif /* PRIo64 */ - - #ifndef PRIx64 - #ifdef _MSC_EXTENSIONS - #define PRIx64 "I64x" - #else - #define PRIx64 "llx" - #endif - #endif - - #ifndef PRIu64 - #ifdef _MSC_EXTENSIONS - #define PRIu64 "I64u" - #else - #define PRIu64 "llu" - #endif - #endif - #if !defined(__cplusplus) #define inline __inline #endif diff --git a/freebsd/contrib/libpcap/savefile.c b/freebsd/contrib/libpcap/savefile.c index 8f6b17c7..6b0cc86e 100644 --- a/freebsd/contrib/libpcap/savefile.c +++ b/freebsd/contrib/libpcap/savefile.c @@ -31,21 +31,13 @@ */ #ifdef HAVE_CONFIG_H -#include "config.h" +#include <config.h> #endif +#include <pcap-types.h> #ifdef _WIN32 -#include <pcap-stdinc.h> -#else /* _WIN32 */ -#if HAVE_INTTYPES_H -#include <inttypes.h> -#elif HAVE_STDINT_H -#include <stdint.h> -#endif -#ifdef HAVE_SYS_BITYPES_H -#include <sys/bitypes.h> -#endif -#include <sys/types.h> +#include <io.h> +#include <fcntl.h> #endif /* _WIN32 */ #include <errno.h> @@ -61,7 +53,7 @@ #endif #include "sf-pcap.h" -#include "sf-pcap-ng.h" +#include "sf-pcapng.h" #ifdef _WIN32 /* @@ -94,7 +86,7 @@ static pcap_t *pcap_fopen_offline(FILE *, char *); #endif static int -sf_getnonblock(pcap_t *p, char *errbuf) +sf_getnonblock(pcap_t *p _U_) { /* * This is a savefile, not a live capture file, so never say @@ -104,7 +96,7 @@ sf_getnonblock(pcap_t *p, char *errbuf) } static int -sf_setnonblock(pcap_t *p, int nonblock, char *errbuf) +sf_setnonblock(pcap_t *p, int nonblock _U_) { /* * This is a savefile, not a live capture file, so reject @@ -120,7 +112,7 @@ sf_setnonblock(pcap_t *p, int nonblock, char *errbuf) } static int -sf_stats(pcap_t *p, struct pcap_stat *ps) +sf_stats(pcap_t *p, struct pcap_stat *ps _U_) { pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "Statistics aren't available from savefiles"); @@ -238,7 +230,7 @@ sf_inject(pcap_t *p, const void *buf _U_, size_t size _U_) * single device? IN, OUT or both? */ static int -sf_setdirection(pcap_t *p, pcap_direction_t d) +sf_setdirection(pcap_t *p, pcap_direction_t d _U_) { pcap_snprintf(p->errbuf, sizeof(p->errbuf), "Setting direction is not supported on savefiles"); @@ -255,22 +247,6 @@ sf_cleanup(pcap_t *p) pcap_freecode(&p->fcode); } -/* -* fopen's safe version on Windows. -*/ -#ifdef _MSC_VER -FILE *fopen_safe(const char *filename, const char* mode) -{ - FILE *fp = NULL; - errno_t errno; - errno = fopen_s(&fp, filename, mode); - if (errno == 0) - return fp; - else - return NULL; -} -#endif - pcap_t * pcap_open_offline_with_tstamp_precision(const char *fname, u_int precision, char *errbuf) @@ -295,14 +271,16 @@ pcap_open_offline_with_tstamp_precision(const char *fname, u_int precision, #endif } else { -#if !defined(_WIN32) && !defined(MSDOS) - fp = fopen(fname, "r"); -#else + /* + * "b" is supported as of C90, so *all* UN*Xes should + * support it, even though it does nothing. It's + * required on Windows, as the file is a binary file + * and must be read in binary mode. + */ fp = fopen(fname, "rb"); -#endif if (fp == NULL) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, "%s: %s", fname, - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "%s", fname); return (NULL); } } @@ -331,14 +309,16 @@ pcap_t* pcap_hopen_offline_with_tstamp_precision(intptr_t osfd, u_int precision, fd = _open_osfhandle(osfd, _O_RDONLY); if ( fd < 0 ) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "_open_osfhandle"); return NULL; } file = _fdopen(fd, "rb"); if ( file == NULL ) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "_fdopen"); return NULL; } @@ -375,7 +355,7 @@ pcap_fopen_offline_with_tstamp_precision(FILE *fp, u_int precision, /* * Read the first 4 bytes of the file; the network analyzer dump - * file formats we support (pcap and pcap-ng), and several other + * file formats we support (pcap and pcapng), and several other * formats we might support in the future (such as snoop, DOS and * Windows Sniffer, and Microsoft Network Monitor) all have magic * numbers that are unique in their first 4 bytes. @@ -383,9 +363,8 @@ pcap_fopen_offline_with_tstamp_precision(FILE *fp, u_int precision, amt_read = fread((char *)&magic, 1, sizeof(magic), fp); if (amt_read != sizeof(magic)) { if (ferror(fp)) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "error reading dump file: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "error reading dump file"); } else { pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, "truncated dump file; tried to read %lu file header bytes, only got %lu", diff --git a/freebsd/contrib/libpcap/scanner.c b/freebsd/contrib/libpcap/scanner.c index aaaa209f..3a477eb5 100644 --- a/freebsd/contrib/libpcap/scanner.c +++ b/freebsd/contrib/libpcap/scanner.c @@ -2,12 +2,34 @@ #line 2 "scanner.l" /* Must come first for _LARGE_FILE_API on AIX. */ #ifdef HAVE_CONFIG_H -#include "config.h" +#include <config.h> #endif +/* + * Must come first to avoid warnings on Windows. + * + * Flex-generated scanners may only include <inttypes.h> if __STDC_VERSION__ + * is defined with a value >= 199901, meaning "full C99", and MSVC may not + * define it with that value, because it isn't 100% C99-compliant, even + * though it has an <inttypes.h> capable of defining everything the Flex + * scanner needs. + * + * We, however, will include it if we know we have an MSVC version that has + * it; this means that we may define the INTn_MAX and UINTn_MAX values in + * scanner.c, and then include <stdint.h>, which may define them differently + * (same value, but different string of characters), causing compiler warnings. + * + * If we include it here, and they're defined, that'll prevent scanner.c + * from defining them. So we include <pcap/pcap-inttypes.h>, to get + * <inttypes.h> if we have it. + */ +#include <pcap/pcap-inttypes.h> + +#include "diag-control.h" -#line 11 "scanner.c" + +#line 33 "scanner.c" #define YY_INT_ALIGNED short int @@ -369,8 +391,8 @@ static void yy_fatal_error (yyconst char msg[] ,yyscan_t yyscanner ) __dead2; *yy_cp = '\0'; \ yyg->yy_c_buf_p = yy_cp; -#define YY_NUM_RULES 157 -#define YY_END_OF_BUFFER 158 +#define YY_NUM_RULES 184 +#define YY_END_OF_BUFFER 185 /* This struct is not used in this scanner, but its presence is necessary. */ struct yy_trans_info @@ -378,143 +400,153 @@ struct yy_trans_info flex_int32_t yy_verify; flex_int32_t yy_nxt; }; -static yyconst flex_int16_t yy_accept[1470] = +static yyconst flex_int16_t yy_accept[1729] = { 0, - 0, 0, 158, 155, 113, 113, 113, 114, 155, 114, - 114, 114, 156, 123, 123, 114, 114, 114, 114, 153, - 153, 155, 153, 153, 153, 153, 153, 153, 153, 153, - 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, - 153, 153, 114, 155, 117, 121, 67, 0, 153, 123, - 0, 153, 153, 153, 0, 125, 119, 116, 118, 115, - 120, 153, 154, 154, 153, 153, 153, 20, 153, 153, - 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, - 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, - 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, - - 153, 7, 153, 34, 35, 153, 153, 153, 153, 153, - 153, 153, 153, 153, 153, 153, 153, 153, 153, 92, - 153, 68, 153, 153, 153, 153, 153, 153, 60, 153, - 153, 153, 153, 86, 153, 153, 153, 153, 153, 153, - 61, 153, 4, 153, 153, 153, 153, 153, 153, 153, - 68, 121, 153, 124, 124, 153, 123, 153, 0, 125, - 123, 125, 125, 125, 153, 153, 153, 67, 5, 153, - 81, 153, 153, 153, 153, 153, 153, 153, 55, 107, - 1, 0, 153, 21, 153, 153, 153, 153, 153, 153, - 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, - - 153, 153, 36, 153, 153, 18, 43, 0, 153, 29, - 153, 25, 70, 153, 153, 79, 37, 153, 100, 153, - 153, 153, 153, 101, 153, 46, 69, 82, 106, 153, - 14, 153, 3, 153, 153, 153, 153, 153, 94, 153, - 153, 26, 153, 105, 153, 108, 38, 2, 153, 42, - 153, 9, 153, 10, 89, 153, 88, 153, 153, 0, - 153, 153, 124, 153, 153, 153, 153, 123, 0, 153, + 0, 0, 185, 182, 113, 113, 113, 114, 182, 114, + 114, 114, 183, 123, 123, 114, 114, 114, 114, 180, + 180, 182, 180, 180, 180, 180, 180, 180, 180, 180, + 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, + 180, 180, 114, 182, 117, 121, 67, 0, 180, 123, + 0, 180, 180, 180, 0, 125, 119, 116, 118, 115, + 120, 180, 181, 181, 180, 180, 180, 20, 180, 180, + 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, + 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, + 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, + + 180, 7, 180, 34, 35, 180, 180, 180, 180, 180, + 180, 180, 180, 180, 180, 180, 180, 180, 180, 92, + 180, 68, 180, 180, 180, 180, 180, 180, 60, 180, + 180, 180, 180, 86, 180, 180, 180, 180, 180, 180, + 61, 180, 4, 180, 180, 180, 180, 180, 180, 180, + 68, 121, 180, 124, 124, 180, 123, 180, 0, 125, + 123, 125, 125, 125, 180, 180, 180, 67, 5, 180, + 81, 180, 180, 180, 180, 180, 180, 180, 55, 107, + 1, 0, 180, 21, 180, 180, 180, 180, 180, 180, + 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, + + 180, 180, 36, 180, 180, 18, 43, 0, 180, 29, + 180, 25, 70, 180, 180, 79, 37, 180, 100, 180, + 180, 180, 180, 101, 180, 46, 69, 82, 106, 180, + 14, 180, 3, 180, 180, 180, 180, 180, 94, 180, + 180, 26, 180, 105, 180, 108, 38, 2, 180, 42, + 180, 9, 180, 10, 89, 180, 88, 180, 180, 0, + 180, 180, 124, 180, 180, 180, 180, 123, 0, 180, 0, 126, 125, 125, 0, 125, 0, 125, 0, 125, - 0, 23, 153, 153, 153, 153, 64, 16, 41, 153, - 39, 153, 153, 153, 30, 153, 98, 153, 153, 153, - - 111, 153, 153, 104, 110, 45, 109, 112, 11, 153, - 12, 13, 153, 153, 153, 32, 78, 153, 62, 3, - 99, 47, 153, 153, 153, 74, 153, 153, 153, 153, - 48, 153, 153, 40, 153, 6, 153, 93, 153, 8, - 95, 153, 153, 0, 153, 53, 73, 15, 153, 124, - 124, 153, 124, 124, 124, 153, 123, 153, 0, 125, - 153, 0, 0, 125, 0, 125, 126, 125, 0, 0, - 0, 0, 125, 125, 125, 125, 125, 0, 153, 56, - 57, 58, 59, 153, 22, 153, 153, 153, 153, 31, - 153, 153, 153, 102, 103, 0, 19, 153, 153, 153, - - 87, 153, 33, 153, 80, 28, 27, 153, 153, 83, - 153, 153, 153, 50, 17, 153, 153, 153, 153, 153, - 153, 153, 153, 153, 153, 153, 153, 153, 153, 0, - 153, 153, 124, 153, 153, 153, 153, 124, 124, 153, - 123, 153, 0, 0, 125, 125, 125, 0, 0, 126, + 0, 23, 180, 180, 180, 180, 64, 16, 41, 180, + 39, 180, 180, 180, 30, 180, 98, 180, 180, 180, + + 111, 180, 180, 104, 110, 45, 109, 112, 11, 180, + 12, 13, 180, 180, 180, 32, 78, 180, 62, 3, + 99, 47, 180, 180, 180, 74, 180, 180, 180, 180, + 48, 180, 180, 40, 180, 6, 180, 93, 180, 8, + 95, 180, 180, 0, 180, 53, 73, 15, 180, 124, + 124, 180, 124, 124, 124, 180, 123, 180, 0, 125, + 180, 0, 0, 125, 0, 125, 126, 125, 0, 0, + 0, 0, 125, 125, 125, 125, 125, 0, 180, 56, + 57, 58, 59, 180, 22, 180, 180, 180, 180, 31, + 180, 180, 180, 102, 103, 0, 19, 180, 180, 180, + + 87, 180, 33, 180, 80, 28, 27, 180, 180, 83, + 180, 180, 180, 50, 17, 180, 180, 180, 180, 180, + 180, 180, 180, 180, 180, 180, 180, 180, 180, 0, + 180, 180, 124, 180, 180, 180, 180, 124, 124, 180, + 123, 180, 0, 0, 125, 125, 125, 0, 0, 126, 125, 125, 126, 125, 0, 0, 125, 125, 125, 125, 125, 0, 0, 0, 0, 125, 125, 0, 125, 0, - 125, 0, 97, 153, 153, 153, 24, 153, 153, 77, - 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, - 153, 153, 70, 153, 153, 153, 153, 153, 153, 153, - - 75, 76, 153, 96, 153, 153, 153, 153, 153, 153, - 153, 153, 153, 153, 153, 153, 153, 153, 124, 124, - 153, 124, 124, 124, 124, 153, 123, 153, 0, 125, - 125, 0, 125, 0, 0, 125, 0, 125, 126, 125, - 0, 0, 0, 125, 125, 0, 125, 126, 125, 0, - 0, 0, 0, 0, 0, 0, 125, 125, 125, 125, - 125, 0, 153, 153, 153, 153, 52, 63, 153, 153, - 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, - 71, 153, 153, 44, 84, 85, 153, 153, 153, 153, - 54, 149, 152, 151, 145, 153, 147, 146, 150, 153, - - 0, 153, 153, 124, 153, 153, 153, 124, 153, 123, - 153, 0, 0, 125, 125, 125, 125, 125, 125, 0, - 0, 126, 125, 125, 125, 0, 0, 125, 125, 125, - 125, 125, 0, 0, 0, 0, 0, 0, 0, 125, - 125, 125, 125, 125, 0, 0, 0, 0, 0, 125, - 125, 0, 125, 0, 125, 0, 153, 153, 153, 153, - 153, 153, 153, 153, 153, 153, 153, 153, 153, 128, - 127, 153, 153, 72, 153, 153, 153, 148, 144, 153, - 153, 124, 124, 124, 124, 153, 123, 153, 0, 125, - 125, 0, 125, 125, 0, 125, 0, 0, 125, 0, + 125, 0, 97, 180, 180, 180, 24, 180, 180, 77, + 180, 180, 180, 180, 180, 180, 180, 180, 180, 0, + 180, 180, 180, 180, 180, 70, 180, 180, 180, 180, + + 180, 180, 180, 75, 76, 180, 96, 180, 180, 180, + 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, + 180, 124, 124, 180, 124, 124, 124, 124, 180, 123, + 180, 0, 125, 125, 0, 125, 0, 0, 125, 0, + 125, 126, 125, 0, 0, 0, 125, 125, 0, 125, + 126, 125, 0, 0, 0, 0, 0, 0, 0, 125, + 125, 125, 125, 125, 0, 180, 180, 180, 180, 52, + 63, 180, 180, 180, 180, 180, 180, 180, 180, 180, + 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, + 180, 180, 71, 180, 180, 44, 84, 85, 180, 180, + + 180, 180, 54, 176, 179, 178, 172, 180, 174, 173, + 177, 180, 0, 180, 180, 124, 180, 180, 180, 124, + 180, 123, 180, 0, 0, 125, 125, 125, 125, 125, + 125, 0, 0, 126, 125, 125, 125, 0, 0, 125, + 125, 125, 125, 125, 0, 0, 0, 0, 0, 0, + 0, 125, 125, 125, 125, 125, 0, 0, 0, 0, + 0, 125, 125, 0, 125, 0, 125, 0, 180, 180, + 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, + 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, + 180, 180, 180, 128, 127, 180, 180, 72, 180, 180, + + 180, 175, 171, 180, 180, 124, 124, 124, 124, 180, + 123, 180, 0, 125, 125, 0, 125, 125, 0, 125, + 0, 0, 125, 0, 125, 126, 125, 0, 0, 0, + 125, 125, 0, 125, 126, 125, 0, 0, 0, 0, + 0, 125, 125, 0, 125, 126, 125, 0, 125, 125, + 0, 0, 0, 0, 0, 0, 0, 125, 125, 125, + 125, 125, 0, 65, 180, 55, 133, 140, 180, 180, + 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, + 180, 180, 180, 180, 180, 180, 145, 144, 180, 66, + 49, 180, 180, 0, 180, 180, 180, 180, 180, 123, + + 180, 0, 0, 125, 125, 125, 125, 125, 125, 125, + 125, 125, 0, 0, 126, 125, 125, 125, 0, 0, + 125, 125, 125, 125, 125, 0, 0, 0, 0, 0, + 0, 0, 125, 125, 125, 125, 125, 0, 125, 125, + 0, 0, 0, 0, 0, 0, 0, 125, 125, 125, + 125, 125, 0, 0, 0, 0, 0, 0, 125, 125, + 0, 125, 0, 125, 0, 90, 180, 180, 180, 180, + 180, 180, 180, 180, 180, 180, 180, 146, 180, 180, + 180, 180, 180, 180, 180, 180, 180, 51, 122, 122, + 124, 124, 180, 123, 180, 0, 125, 125, 0, 125, + 125, 0, 125, 125, 0, 125, 0, 122, 125, 0, 125, 126, 125, 0, 0, 0, 125, 125, 0, 125, 126, 125, 0, 0, 0, 0, 0, 125, 125, 0, - 125, 126, 125, 0, 125, 125, 0, 0, 0, 0, - 0, 0, 0, 125, 125, 125, 125, 125, 0, 65, - 153, 55, 133, 140, 153, 153, 153, 153, 153, 153, - 153, 153, 153, 66, 49, 153, 153, 0, 153, 153, - 153, 153, 153, 123, 153, 0, 0, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 0, 0, 126, 125, - 125, 125, 0, 0, 125, 125, 125, 125, 125, 0, + 125, 126, 125, 0, 0, 0, 0, 0, 0, 125, + 125, 0, 125, 126, 125, 0, 125, 125, 125, 0, 0, 0, 0, 0, 0, 0, 125, 125, 125, 125, + 125, 0, 180, 180, 180, 180, 180, 180, 180, 180, + 138, 180, 180, 180, 180, 180, 180, 180, 180, 180, + 180, 180, 91, 122, 122, 124, 180, 122, 122, 0, + 0, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 0, 125, 125, 0, 0, 0, 0, 0, 0, + 125, 125, 125, 0, 122, 126, 125, 125, 125, 0, 0, 125, 125, 125, 125, 125, 0, 0, 0, 0, - 0, 0, 125, 125, 0, 125, 0, 125, 0, 90, - 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, - 153, 51, 122, 122, 124, 124, 153, 123, 153, 0, - 125, 125, 0, 125, 125, 0, 125, 125, 0, 125, - 0, 122, 125, 0, 125, 126, 125, 0, 0, 0, - 125, 125, 0, 125, 126, 125, 0, 0, 0, 0, - 0, 125, 125, 0, 125, 126, 125, 0, 0, 0, + 0, 0, 0, 125, 125, 125, 125, 125, 0, 125, + 125, 0, 0, 0, 0, 0, 0, 0, 125, 125, + 125, 125, 125, 0, 125, 125, 125, 0, 0, 0, + 0, 0, 0, 0, 125, 125, 125, 125, 125, 0, + 0, 0, 0, 0, 0, 125, 125, 0, 125, 0, + 125, 0, 180, 180, 180, 142, 180, 180, 180, 180, + 180, 180, 180, 130, 180, 180, 180, 180, 180, 180, + 180, 180, 180, 180, 124, 180, 123, 0, 125, 125, + + 0, 125, 125, 0, 125, 125, 0, 125, 125, 0, + 125, 0, 0, 0, 125, 0, 0, 125, 126, 125, 0, 0, 0, 125, 125, 0, 125, 126, 125, 0, - + 0, 0, 0, 0, 125, 125, 0, 125, 126, 125, + 0, 0, 0, 0, 0, 0, 125, 125, 0, 125, + 126, 125, 0, 0, 0, 0, 0, 0, 125, 125, + 0, 125, 126, 125, 0, 125, 125, 125, 0, 0, + 0, 0, 0, 0, 0, 125, 125, 125, 125, 125, + 0, 180, 180, 180, 180, 132, 180, 180, 180, 136, + 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, + + 180, 180, 180, 122, 0, 0, 125, 125, 125, 125, + 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, + 125, 0, 0, 0, 126, 0, 0, 125, 0, 0, 125, 125, 125, 0, 0, 0, 0, 0, 0, 0, - 125, 125, 125, 125, 125, 0, 153, 153, 153, 153, - 153, 153, 153, 153, 138, 153, 91, 122, 122, 124, - 153, 122, 122, 0, 0, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 0, 122, 126, - 125, 125, 125, 0, 0, 125, 125, 125, 125, 125, + 125, 125, 125, 0, 125, 125, 0, 0, 0, 0, + 0, 0, 0, 125, 125, 125, 0, 125, 125, 125, 0, 0, 0, 0, 0, 0, 0, 125, 125, 125, - 125, 125, 0, 125, 125, 0, 0, 0, 0, 0, - 0, 0, 125, 125, 125, 125, 125, 0, 125, 125, - 125, 0, 0, 0, 0, 0, 0, 0, 125, 125, - - 125, 125, 125, 0, 0, 0, 0, 0, 0, 125, - 125, 0, 125, 0, 125, 0, 153, 153, 153, 142, - 153, 153, 153, 153, 153, 153, 153, 130, 124, 153, - 123, 0, 125, 125, 0, 125, 125, 0, 125, 125, - 0, 125, 125, 0, 125, 0, 0, 0, 125, 0, - 0, 125, 126, 125, 0, 0, 0, 125, 125, 0, - 125, 126, 125, 0, 0, 0, 0, 0, 125, 125, - 0, 125, 126, 125, 0, 0, 0, 0, 0, 0, - 125, 125, 0, 125, 126, 125, 0, 0, 0, 0, - 0, 0, 125, 125, 0, 125, 126, 125, 0, 125, - - 125, 125, 0, 0, 0, 0, 0, 0, 0, 125, - 125, 125, 125, 125, 0, 153, 153, 153, 153, 132, - 153, 153, 153, 136, 153, 122, 0, 0, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 0, 0, 0, 126, 0, 0, 125, - 0, 0, 125, 125, 125, 0, 0, 0, 0, 0, - 0, 0, 125, 125, 125, 0, 125, 125, 0, 0, - 0, 0, 0, 0, 0, 125, 125, 125, 0, 125, - 125, 125, 0, 0, 0, 0, 0, 0, 0, 125, - 125, 125, 0, 125, 125, 125, 0, 0, 0, 0, + 0, 125, 125, 125, 0, 0, 0, 0, 0, 0, + 0, 125, 125, 125, 0, 0, 0, 0, 0, 0, + 125, 125, 0, 125, 0, 125, 0, 129, 141, 143, - 0, 0, 0, 125, 125, 125, 0, 0, 0, 0, - 0, 0, 125, 125, 0, 125, 0, 125, 0, 129, - 141, 143, 137, 153, 153, 153, 153, 0, 0, 125, + 137, 180, 180, 180, 180, 180, 180, 180, 180, 180, + 180, 180, 180, 155, 180, 180, 180, 0, 0, 125, 0, 125, 0, 125, 125, 0, 125, 125, 0, 125, 125, 0, 125, 125, 0, 125, 0, 0, 0, 0, 125, 125, 0, 125, 0, 0, 125, 125, 125, 0, @@ -522,25 +554,44 @@ static yyconst flex_int16_t yy_accept[1470] = 0, 125, 125, 125, 0, 0, 0, 0, 0, 125, 125, 125, 0, 0, 0, 0, 0, 125, 125, 125, 125, 125, 125, 0, 0, 0, 0, 0, 0, 0, + 125, 125, 125, 0, 180, 180, 180, 180, 180, 180, - 125, 125, 125, 0, 153, 153, 153, 153, 0, 0, - 0, 125, 125, 125, 125, 125, 125, 0, 0, 0, - 0, 125, 125, 0, 0, 0, 0, 125, 125, 125, - 0, 0, 0, 0, 0, 125, 125, 125, 125, 0, - 0, 0, 0, 0, 125, 125, 125, 125, 0, 0, - 0, 0, 0, 125, 125, 125, 125, 0, 0, 0, - 0, 0, 125, 0, 0, 0, 0, 0, 125, 125, - 125, 153, 153, 153, 139, 125, 125, 125, 125, 125, - 125, 125, 125, 0, 0, 0, 0, 125, 125, 0, - 0, 125, 0, 0, 0, 125, 0, 0, 0, 125, + 147, 180, 180, 180, 180, 180, 180, 180, 180, 180, + 180, 0, 0, 0, 125, 125, 125, 125, 125, 125, + 0, 0, 0, 0, 125, 125, 0, 0, 0, 0, + 125, 125, 125, 0, 0, 0, 0, 0, 125, 125, + 125, 125, 0, 0, 0, 0, 0, 125, 125, 125, + 125, 0, 0, 0, 0, 0, 125, 125, 125, 125, + 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, + 0, 125, 125, 125, 180, 180, 180, 139, 180, 180, + 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, + 180, 125, 125, 125, 125, 125, 125, 125, 125, 0, - 0, 0, 0, 125, 0, 0, 0, 125, 125, 125, - 125, 0, 0, 0, 0, 0, 125, 134, 153, 131, - 125, 0, 0, 125, 125, 0, 125, 125, 125, 0, - 125, 125, 125, 0, 125, 125, 125, 0, 125, 125, - 125, 0, 0, 0, 0, 125, 135, 125, 125, 0, - 0, 0, 0, 0, 0, 125, 125, 125, 0, 0, - 125, 125, 125, 125, 125, 0, 125, 125, 0 + 0, 0, 0, 125, 125, 0, 0, 125, 0, 0, + 0, 125, 0, 0, 0, 125, 0, 0, 0, 125, + 0, 0, 0, 125, 125, 125, 125, 0, 0, 0, + 0, 0, 125, 134, 180, 131, 180, 180, 180, 180, + 180, 180, 180, 180, 180, 180, 180, 156, 180, 125, + 0, 0, 125, 125, 0, 125, 125, 125, 0, 125, + 125, 125, 0, 125, 125, 125, 0, 125, 125, 125, + 0, 0, 0, 0, 125, 135, 180, 180, 180, 180, + 180, 180, 180, 180, 180, 180, 152, 180, 125, 125, + 0, 0, 0, 0, 0, 0, 125, 125, 125, 0, + + 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, + 180, 151, 0, 125, 125, 125, 125, 125, 0, 167, + 180, 180, 180, 180, 180, 180, 180, 154, 180, 180, + 125, 125, 166, 180, 180, 180, 180, 180, 180, 153, + 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, + 180, 180, 180, 180, 180, 180, 180, 180, 180, 180, + 180, 180, 180, 165, 180, 180, 180, 180, 180, 180, + 180, 180, 180, 180, 180, 180, 164, 180, 180, 180, + 180, 180, 170, 180, 180, 180, 180, 180, 180, 180, + 180, 180, 180, 157, 180, 180, 180, 180, 180, 150, + + 180, 180, 168, 180, 180, 180, 180, 180, 180, 148, + 180, 169, 180, 163, 180, 180, 180, 180, 158, 180, + 160, 180, 180, 162, 159, 149, 161, 0 } ; static yyconst flex_int32_t yy_ec[256] = @@ -585,231 +636,259 @@ static yyconst flex_int32_t yy_meta[54] = 9, 4, 3 } ; -static yyconst flex_int16_t yy_base[1930] = +static yyconst flex_int16_t yy_base[2189] = { 0, - 0, 0, 3903, 53, 7432, 7432, 57, 3881, 64, 81, - 3893, 7432, 7432, 99, 30, 151, 46, 3879, 52, 168, + 0, 0, 5455, 53, 7856, 7856, 57, 5433, 64, 81, + 5445, 7856, 7856, 99, 30, 151, 46, 5430, 52, 168, 210, 168, 160, 44, 125, 60, 31, 58, 132, 170, - 214, 217, 229, 59, 170, 222, 237, 243, 250, 3867, - 255, 3858, 3843, 300, 7432, 0, 7432, 316, 339, 363, - 3876, 387, 0, 394, 0, 428, 7432, 7432, 7432, 7432, - 7432, 296, 316, 0, 3834, 3831, 3845, 0, 3844, 3832, - 3844, 3841, 3829, 3809, 3810, 3813, 3812, 3811, 3819, 3804, - 3817, 3785, 116, 3795, 3798, 3783, 3781, 3785, 3792, 3781, - 3772, 3762, 3761, 56, 68, 3766, 3764, 80, 3767, 3762, - - 3771, 68, 166, 0, 0, 15, 122, 3744, 3753, 189, - 3737, 3735, 3738, 3739, 3729, 3723, 3714, 3713, 3719, 0, - 3727, 0, 3709, 3715, 3709, 3695, 3695, 3695, 128, 3706, - 3688, 3698, 3676, 159, 3672, 201, 3670, 38, 3669, 3681, - 0, 3666, 0, 3665, 3664, 3654, 3661, 3652, 3643, 3657, - 7432, 7432, 453, 477, 236, 518, 542, 566, 3665, 573, - 3657, 597, 248, 3648, 3623, 3628, 3619, 0, 0, 3623, - 0, 3632, 3627, 3601, 3600, 3601, 3598, 3598, 3605, 0, - 0, 3600, 3575, 0, 3587, 3582, 3570, 3584, 3582, 3585, - 3582, 3566, 3544, 3541, 3556, 3539, 3542, 3537, 3539, 3552, - - 3523, 3522, 0, 3527, 3521, 0, 0, 3523, 3513, 0, - 3525, 0, 3522, 3501, 3488, 0, 0, 3477, 0, 3486, - 3493, 244, 3474, 0, 3472, 3488, 0, 3483, 0, 3471, - 0, 3453, 3457, 3451, 3453, 3458, 3451, 3447, 0, 3431, - 3444, 0, 3432, 0, 3430, 0, 0, 0, 3427, 0, - 153, 156, 3437, 0, 0, 3413, 0, 3410, 3411, 637, - 3440, 660, 684, 3437, 691, 500, 281, 715, 3428, 739, - 3427, 3411, 747, 290, 3410, 3409, 507, 788, 811, 3408, - 0, 0, 3382, 350, 3385, 3380, 0, 0, 0, 3357, - 0, 3356, 3357, 3338, 0, 3338, 0, 3332, 3317, 3318, - - 0, 3316, 3315, 0, 0, 0, 0, 0, 614, 3322, - 0, 0, 3328, 3310, 3301, 0, 0, 3278, 0, 0, - 0, 0, 3293, 3284, 3290, 0, 3281, 3268, 3284, 3257, - 3252, 3262, 3252, 0, 3227, 0, 3225, 0, 275, 0, - 0, 3212, 3207, 817, 3219, 0, 0, 0, 856, 880, - 293, 921, 3224, 3223, 405, 944, 968, 992, 3214, 999, - 621, 3213, 3210, 1022, 770, 1046, 1069, 3209, 0, 3198, - 427, 428, 1093, 3176, 1117, 333, 3175, 3182, 3150, 0, - 0, 0, 0, 3143, 0, 3157, 3156, 3129, 3128, 0, - 3125, 3117, 3114, 0, 0, 1136, 0, 3104, 3083, 3101, - - 0, 3090, 0, 3094, 3086, 0, 0, 3094, 3076, 291, - 3075, 3093, 294, 3090, 0, 3053, 3042, 3056, 3049, 3044, - 3041, 3021, 3039, 3031, 3018, 3018, 2987, 2993, 3008, 1173, - 3026, 1196, 1220, 3023, 1227, 777, 285, 1251, 334, 1291, - 1314, 1338, 3014, 3013, 1346, 336, 2997, 2996, 2995, 2993, - 1387, 363, 2991, 2990, 515, 631, 1428, 2989, 1452, 364, - 2973, 2980, 2967, 840, 0, 371, 2966, 903, 1493, 1516, - 2965, 0, 0, 2922, 2937, 2915, 0, 2924, 2876, 0, - 2882, 2895, 2879, 2893, 2892, 380, 2878, 427, 2863, 2871, - 2859, 2869, 0, 2859, 2869, 2860, 2865, 2849, 2838, 2837, - - 0, 0, 2841, 0, 2836, 2827, 2840, 2839, 2819, 2811, - 2822, 2813, 2807, 2804, 2809, 2814, 2798, 1541, 1565, 423, - 1606, 2821, 2820, 633, 1630, 1654, 1661, 1685, 2811, 1692, - 1716, 1739, 2810, 2808, 2807, 1762, 910, 1786, 1809, 2806, - 0, 1140, 0, 461, 2791, 1147, 1833, 1856, 2790, 0, - 785, 824, 2797, 485, 853, 856, 1880, 2788, 1904, 424, - 2787, 2793, 405, 2757, 2762, 2744, 0, 0, 2749, 2751, - 2737, 2737, 2748, 2731, 2730, 2723, 2715, 2716, 2727, 2726, - 0, 2716, 2710, 0, 0, 0, 2723, 2704, 2709, 2696, - 0, 0, 0, 0, 0, 2700, 0, 0, 0, 2689, - - 1944, 2722, 1967, 1991, 2720, 1998, 425, 2022, 2046, 2053, - 2077, 2697, 2696, 2085, 452, 2695, 2126, 476, 2694, 2693, - 2691, 2690, 2167, 512, 2689, 918, 938, 2208, 2673, 2232, - 513, 2672, 2679, 1167, 1169, 2678, 2676, 1261, 1262, 2273, - 2666, 2297, 517, 2650, 2657, 1276, 0, 1283, 0, 486, - 2648, 1369, 2338, 2361, 2647, 0, 2384, 452, 57, 497, - 150, 641, 205, 610, 2618, 258, 291, 570, 783, 2616, - 2615, 800, 2614, 2598, 1168, 893, 498, 2597, 2596, 2422, - 2459, 2495, 2531, 518, 2555, 551, 2563, 2587, 2623, 2594, - 2618, 2641, 2621, 2665, 2688, 2620, 2619, 2603, 2711, 1376, - - 2735, 2758, 2602, 0, 1410, 0, 582, 2601, 1417, 2782, - 2805, 2600, 0, 1475, 0, 1482, 0, 699, 2599, 1588, - 2829, 2852, 2597, 0, 570, 1595, 2604, 1425, 1490, 2603, - 2602, 1510, 1541, 2876, 2572, 2900, 572, 2571, 2578, 590, - 652, 615, 636, 787, 1427, 1914, 1492, 706, 709, 849, - 2095, 707, 937, 631, 803, 1898, 2122, 2942, 872, 2965, - 873, 2988, 1931, 3012, 3036, 2569, 2568, 3044, 660, 2566, - 3085, 661, 2565, 3126, 690, 2564, 2563, 2540, 2539, 3167, - 717, 2538, 1603, 1733, 3208, 2537, 3232, 718, 2536, 2542, - 1940, 1941, 2541, 2540, 2095, 2096, 3273, 2531, 3297, 787, - - 2484, 2490, 851, 2112, 2476, 2123, 2142, 2475, 2473, 2143, - 2162, 3338, 2462, 3362, 920, 2428, 2435, 0, 2190, 0, - 2255, 0, 756, 2414, 2262, 3403, 3426, 2392, 0, 936, - 960, 962, 1014, 1897, 1015, 1605, 1016, 1040, 1943, 1062, - 1922, 938, 3451, 3474, 3498, 967, 3538, 3562, 3586, 2387, - 3593, 3617, 3640, 2386, 3664, 3687, 2385, 3711, 3734, 2369, - 2368, 2367, 3757, 2320, 3781, 3804, 2366, 0, 2327, 0, - 1008, 2364, 2407, 3828, 3851, 2363, 0, 2445, 0, 2482, - 0, 1055, 2347, 2502, 3875, 3898, 2344, 0, 0, 2509, - 0, 2516, 0, 1078, 2343, 2523, 3922, 3945, 2341, 0, - - 0, 969, 2923, 2348, 2164, 2184, 2347, 2314, 2203, 2205, - 3969, 2305, 3993, 1045, 2304, 2308, 2337, 2398, 2399, 2272, - 1110, 2142, 2400, 1085, 2336, 1109, 1111, 1165, 1167, 4035, - 4059, 4068, 1188, 2299, 2282, 4086, 1066, 2280, 4127, 1119, - 2279, 4168, 1146, 2278, 4209, 1149, 2275, 2274, 2240, 4249, - 4273, 1196, 2239, 2270, 2378, 4314, 2234, 4338, 1197, 2217, - 2222, 2453, 2454, 2221, 2218, 2635, 2682, 4379, 2198, 4403, - 1198, 2193, 2200, 1226, 2930, 2199, 2938, 2939, 2180, 2165, - 2942, 3054, 4444, 2153, 4468, 1227, 2150, 2155, 0, 1228, - 3068, 2154, 3081, 3082, 2153, 2152, 3101, 3102, 4509, 2142, - - 4533, 1285, 2114, 2120, 0, 3149, 0, 3156, 0, 1235, - 2103, 3190, 4574, 4597, 2094, 0, 3081, 3242, 3243, 1189, - 2438, 2402, 1306, 3226, 2959, 1307, 1446, 1305, 4622, 4646, - 4655, 2093, 4672, 4696, 4719, 2092, 4743, 4766, 2090, 4790, - 4813, 2062, 4837, 4860, 2060, 2059, 4884, 1315, 2058, 2056, - 3197, 4925, 2055, 2054, 0, 3260, 0, 1300, 2030, 3320, - 4949, 2029, 2028, 0, 3327, 0, 3385, 0, 1396, 2027, - 3392, 4973, 2025, 2024, 0, 0, 3521, 0, 3528, 0, - 1525, 2023, 4016, 4997, 2005, 2003, 0, 0, 4023, 0, - 4075, 0, 1549, 2001, 4109, 5021, 1922, 1864, 0, 0, - - 1316, 4116, 1870, 3205, 3268, 1847, 1824, 3335, 3400, 5045, - 1814, 5069, 1317, 1811, 1803, 1708, 1709, 1731, 1557, 1385, - 1780, 4125, 1624, 1386, 2440, 5111, 1792, 5128, 5152, 1427, - 1791, 5193, 1429, 1768, 5234, 1489, 1767, 5275, 1492, 1748, - 5316, 1538, 1747, 1746, 4153, 5357, 1744, 1664, 0, 1663, - 3420, 3451, 5381, 1636, 1608, 1615, 3537, 3634, 1582, 1575, - 3681, 3728, 5405, 1565, 1550, 1551, 1540, 4191, 1531, 4137, - 4138, 1527, 1508, 4165, 4185, 5429, 1499, 1494, 1462, 0, - 1600, 4232, 1442, 4204, 4206, 1438, 1437, 4226, 4248, 5453, - 1392, 1390, 1397, 0, 1605, 4296, 1363, 4290, 4309, 1362, - - 1361, 4311, 4348, 5477, 1351, 1299, 1264, 0, 4362, 0, - 4369, 0, 1669, 1252, 4426, 5501, 0, 1251, 0, 1621, - 1710, 1754, 1778, 1779, 2013, 4462, 4478, 5525, 1607, 0, - 1236, 5566, 0, 75, 5590, 0, 102, 5614, 0, 137, - 5638, 0, 143, 5662, 0, 195, 4420, 4441, 5686, 198, - 217, 250, 314, 317, 4495, 0, 1670, 341, 348, 4556, - 0, 4563, 0, 1842, 372, 412, 0, 4662, 0, 4907, - 0, 1865, 414, 431, 0, 4914, 0, 5092, 0, 1953, - 435, 462, 0, 5099, 0, 5118, 0, 1976, 548, 551, - 0, 1660, 5175, 583, 4504, 4571, 608, 629, 4591, 4671, - - 5710, 644, 0, 654, 2457, 1803, 1826, 1849, 5734, 694, - 752, 0, 0, 0, 0, 0, 0, 5182, 0, 2006, - 754, 755, 0, 4713, 4760, 797, 798, 0, 1661, 5216, - 803, 4807, 4854, 826, 828, 0, 0, 1662, 5223, 850, - 4922, 5127, 888, 895, 0, 0, 1689, 5257, 897, 5231, - 5251, 930, 931, 0, 0, 1691, 5298, 976, 5270, 5272, - 985, 1010, 0, 0, 5339, 0, 5346, 0, 2007, 1004, - 0, 3101, 3003, 2224, 1827, 0, 7432, 0, 0, 0, - 0, 0, 0, 5292, 5354, 1035, 1056, 0, 7432, 5548, - 0, 7432, 0, 5555, 0, 7432, 0, 5757, 0, 7432, - - 0, 5764, 0, 7432, 0, 5771, 0, 7432, 0, 1692, - 5778, 1060, 5563, 5786, 1083, 2061, 0, 1850, 3270, 1872, - 0, 5786, 0, 1855, 5800, 1103, 0, 1877, 5807, 1104, - 0, 1879, 5814, 1132, 0, 1880, 5826, 1155, 0, 1938, - 5833, 1156, 0, 5840, 0, 7432, 1960, 1943, 5847, 1161, - 0, 0, 0, 0, 0, 0, 1968, 5854, 1213, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 7432, 5872, - 5880, 5884, 5887, 5890, 5893, 5896, 5899, 5902, 5905, 5908, - 5911, 5914, 5917, 5920, 5923, 5926, 5929, 5932, 5936, 5940, - 5943, 5946, 5949, 5952, 5955, 5958, 5961, 5964, 5968, 5972, - - 5975, 5978, 5982, 5984, 5987, 5990, 5993, 5996, 5999, 6002, - 6005, 6008, 6012, 6014, 6017, 6021, 6026, 6030, 6033, 6037, - 6040, 6043, 6046, 6049, 6052, 6055, 6058, 6062, 6066, 6069, - 6073, 6077, 6082, 6086, 6088, 6092, 6095, 6099, 6102, 6105, - 6109, 6111, 6114, 6117, 6120, 6123, 6126, 6129, 6132, 6135, - 6138, 6142, 6144, 6147, 6150, 6153, 6157, 6159, 6162, 6165, - 6170, 6174, 6179, 6183, 6185, 6189, 6192, 6196, 6201, 6205, - 6208, 6211, 6214, 6217, 6220, 6223, 6226, 6230, 6234, 6237, - 6241, 6245, 6250, 6254, 6256, 6260, 6263, 6267, 6270, 6275, - 6279, 6284, 6288, 6290, 6294, 6297, 6301, 6304, 6307, 6310, - - 6314, 6316, 6319, 6324, 6328, 6331, 6334, 6337, 6340, 6343, - 6346, 6349, 6352, 6356, 6358, 6361, 6364, 6367, 6371, 6373, - 6376, 6379, 6382, 6385, 6389, 6391, 6394, 6397, 6400, 6405, - 6409, 6414, 6418, 6420, 6424, 6427, 6431, 6436, 6440, 6443, - 6446, 6449, 6452, 6455, 6458, 6461, 6465, 6469, 6472, 6476, - 6480, 6485, 6489, 6491, 6495, 6498, 6502, 6505, 6510, 6514, - 6519, 6523, 6525, 6529, 6532, 6536, 6539, 6542, 6547, 6551, - 6556, 6560, 6562, 6566, 6569, 6573, 6576, 6579, 6582, 6586, - 6588, 6591, 6596, 6600, 6603, 6606, 6609, 6612, 6615, 6618, - 6621, 6624, 6627, 6630, 6633, 6637, 6639, 6642, 6645, 6648, - - 6651, 6655, 6657, 6660, 6663, 6666, 6669, 6672, 6676, 6678, - 6681, 6684, 6687, 6690, 6693, 6697, 6699, 6702, 6705, 6708, - 6711, 6716, 6720, 6725, 6729, 6731, 6735, 6738, 6742, 6747, - 6751, 6754, 6757, 6760, 6763, 6766, 6769, 6772, 6775, 6778, - 6782, 6786, 6789, 6793, 6797, 6802, 6806, 6808, 6812, 6815, - 6819, 6822, 6827, 6831, 6836, 6840, 6842, 6846, 6849, 6853, - 6856, 6859, 6864, 6868, 6873, 6877, 6879, 6883, 6886, 6890, - 6893, 6896, 6901, 6905, 6910, 6914, 6916, 6920, 6923, 6927, - 6930, 6933, 6936, 6940, 6942, 6945, 6948, 6953, 6957, 6960, - 6963, 6966, 6969, 6972, 6975, 6978, 6981, 6984, 6987, 6990, - - 6994, 6998, 7001, 7004, 7008, 7011, 7014, 7018, 7020, 7023, - 7026, 7030, 7032, 7035, 7038, 7041, 7045, 7047, 7050, 7053, - 7056, 7060, 7062, 7065, 7068, 7071, 7075, 7077, 7080, 7083, - 7088, 7092, 7097, 7101, 7103, 7107, 7110, 7114, 7119, 7123, - 7126, 7129, 7132, 7135, 7138, 7141, 7144, 7147, 7151, 7153, - 7156, 7160, 7165, 7169, 7170, 7173, 7178, 7182, 7187, 7191, - 7192, 7195, 7198, 7203, 7207, 7212, 7216, 7217, 7220, 7223, - 7228, 7232, 7237, 7241, 7242, 7245, 7248, 7253, 7257, 7262, - 7266, 7267, 7270, 7273, 7276, 7280, 7282, 7287, 7291, 7294, - 7297, 7300, 7303, 7306, 7309, 7313, 7318, 7322, 7323, 7326, - - 7329, 7332, 7335, 7338, 7341, 7344, 7347, 7350, 7353, 7358, - 7362, 7365, 7368, 7371, 7375, 7379, 7383, 7387, 7391, 7394, - 7397, 7401, 7404, 7407, 7410, 7413, 7416, 7420, 7423 + 214, 217, 229, 59, 170, 222, 237, 243, 250, 5418, + 255, 5410, 5380, 300, 7856, 0, 7856, 316, 339, 363, + 5413, 387, 0, 394, 0, 428, 7856, 7856, 7856, 7856, + 7856, 296, 316, 0, 5386, 5383, 5397, 0, 5395, 5383, + 5396, 5378, 5366, 5360, 5361, 5362, 5360, 5344, 5352, 5337, + 5350, 5331, 116, 5341, 5312, 5297, 5295, 5299, 5305, 5294, + 5299, 5279, 5278, 56, 68, 5263, 5261, 80, 5265, 5259, + + 5267, 68, 166, 0, 0, 15, 122, 5255, 5264, 189, + 5238, 5236, 5219, 5222, 5212, 5219, 5209, 5208, 5214, 0, + 5212, 0, 5195, 5181, 5175, 5176, 5175, 5174, 128, 5185, + 5168, 5167, 5160, 159, 5147, 201, 5144, 38, 5137, 5149, + 0, 5133, 0, 5112, 5111, 5094, 5090, 5078, 5048, 5062, + 7856, 7856, 453, 477, 236, 518, 542, 566, 5071, 573, + 5078, 597, 248, 5054, 5029, 5034, 5025, 0, 0, 5030, + 0, 5038, 5033, 5022, 5006, 5007, 5004, 5005, 5012, 0, + 0, 5006, 4996, 0, 5008, 4988, 4976, 4990, 4989, 4992, + 4988, 4972, 4971, 4953, 4969, 4952, 4956, 4951, 4952, 4965, + + 4950, 4934, 0, 4939, 4933, 0, 0, 4937, 4927, 0, + 4938, 0, 4935, 4907, 4912, 0, 0, 4901, 0, 4909, + 4917, 244, 4899, 0, 4887, 4882, 0, 4877, 0, 4880, + 0, 4861, 4864, 4857, 4845, 4850, 4843, 4839, 0, 4837, + 4849, 0, 4838, 0, 4837, 0, 0, 0, 4819, 0, + 153, 156, 4830, 0, 0, 4821, 0, 4818, 4818, 637, + 4847, 660, 684, 4830, 691, 500, 281, 715, 4821, 739, + 4820, 4819, 747, 290, 4818, 4816, 507, 788, 811, 4815, + 0, 0, 4775, 350, 4778, 4783, 0, 0, 0, 4781, + 0, 4777, 4762, 4746, 0, 4746, 0, 4740, 4739, 4740, + + 0, 4722, 4721, 0, 0, 0, 0, 0, 614, 4728, + 0, 0, 4736, 4715, 4700, 0, 0, 4698, 0, 0, + 0, 0, 4713, 4704, 4710, 0, 4703, 4706, 4707, 4679, + 4675, 4667, 4639, 0, 4632, 0, 4607, 0, 275, 0, + 0, 4599, 4592, 817, 4585, 0, 0, 0, 856, 880, + 293, 921, 4610, 4608, 405, 944, 968, 992, 4598, 999, + 621, 4581, 4579, 1022, 770, 1046, 1069, 4562, 0, 4553, + 427, 428, 1093, 4552, 1117, 333, 4551, 4550, 4519, 0, + 0, 0, 0, 4491, 0, 4504, 4502, 4469, 4460, 0, + 4477, 4472, 4463, 0, 0, 1136, 293, 4453, 4420, 4437, + + 0, 4427, 0, 4429, 4422, 0, 0, 4420, 4394, 292, + 4393, 4411, 387, 4408, 0, 4390, 4382, 4394, 4368, 4362, + 4372, 4350, 4357, 4338, 4317, 4318, 4310, 4278, 4292, 1173, + 4310, 1196, 1220, 4305, 1227, 777, 285, 1251, 334, 1291, + 1314, 1338, 4294, 4293, 1346, 336, 4291, 4256, 4255, 4254, + 1387, 363, 4252, 4251, 515, 631, 1428, 4250, 1452, 364, + 4240, 4247, 4218, 840, 0, 371, 4214, 903, 1493, 1516, + 4212, 0, 0, 4184, 4200, 4149, 0, 4158, 4140, 0, + 4145, 4158, 4143, 4126, 4125, 380, 4109, 427, 4108, 1121, + 4106, 4095, 4082, 4067, 4076, 0, 4066, 4077, 4025, 4028, + + 4026, 4015, 4014, 0, 0, 4018, 0, 3981, 3973, 3986, + 3984, 3976, 3949, 3960, 3929, 3923, 3918, 3909, 3888, 3885, + 1541, 1565, 423, 1606, 3892, 3888, 633, 1630, 1654, 1661, + 1685, 3877, 1692, 1716, 1739, 3876, 3857, 3856, 1762, 910, + 1786, 1809, 3852, 0, 1274, 0, 461, 3837, 1281, 1833, + 1856, 3833, 0, 785, 824, 3820, 485, 853, 856, 1880, + 3810, 1904, 424, 3809, 3798, 529, 3762, 3765, 3739, 0, + 0, 3742, 3729, 3715, 3691, 3703, 3683, 3666, 3672, 3664, + 3643, 3632, 3633, 3619, 3601, 56, 504, 646, 3609, 3575, + 3584, 3582, 0, 3573, 3567, 0, 0, 0, 3580, 3552, + + 3533, 3519, 0, 0, 0, 0, 0, 3523, 0, 0, + 0, 3512, 1944, 3547, 1967, 1991, 3514, 1998, 325, 2022, + 2046, 2053, 2077, 3505, 3474, 2085, 452, 3434, 2126, 476, + 3393, 3386, 3385, 3365, 2167, 513, 3345, 918, 938, 2208, + 3344, 2232, 542, 3343, 3348, 1142, 1149, 3347, 3314, 1167, + 1169, 2273, 3305, 2297, 543, 3304, 3310, 1369, 0, 1376, + 0, 486, 3282, 1410, 2338, 2361, 3278, 0, 2384, 374, + 115, 389, 150, 672, 205, 610, 3235, 550, 403, 313, + 783, 553, 761, 522, 800, 451, 578, 893, 633, 829, + 571, 1108, 1112, 3231, 3228, 1123, 3206, 3189, 1130, 1128, + + 593, 3186, 3185, 2422, 2459, 2495, 2531, 634, 2555, 258, + 2563, 2587, 3194, 2594, 2618, 2641, 3193, 2665, 2688, 3191, + 3188, 3185, 2711, 1417, 2735, 2758, 3184, 0, 1475, 0, + 723, 3172, 1482, 2782, 2805, 3147, 0, 1588, 0, 1595, + 0, 724, 3146, 1927, 2829, 2852, 3135, 0, 661, 1934, + 3122, 1290, 1425, 3114, 3111, 1490, 1510, 2876, 3101, 2900, + 662, 3090, 3061, 636, 803, 653, 1290, 1427, 1492, 2095, + 1604, 708, 849, 937, 2122, 872, 1605, 1944, 804, 1187, + 1242, 2099, 936, 960, 1532, 2101, 874, 938, 1015, 962, + 1014, 2124, 2163, 2942, 1016, 2965, 1038, 2988, 2154, 3012, + + 3036, 3052, 3051, 3044, 686, 3049, 3085, 967, 3047, 3126, + 969, 3046, 3045, 3020, 3017, 3167, 970, 2967, 1541, 1733, + 3208, 2966, 3232, 1022, 2947, 2953, 2096, 2163, 2951, 2950, + 2164, 2182, 3273, 2909, 3297, 1042, 2908, 2915, 1043, 2255, + 2913, 2183, 2184, 2911, 2892, 2205, 2249, 3338, 2882, 3362, + 1045, 2881, 2886, 0, 2263, 0, 2320, 0, 756, 2857, + 2327, 3403, 3426, 2853, 0, 1061, 1062, 1087, 1111, 2337, + 1188, 1943, 1189, 1244, 2376, 1534, 2377, 2336, 1109, 1243, + 2402, 2013, 1306, 2096, 2378, 1446, 2375, 1245, 3451, 3474, + 3498, 1066, 3538, 3562, 3586, 2834, 3593, 3617, 3640, 2830, + + 3664, 3687, 2811, 3711, 3734, 2810, 2806, 2791, 3757, 2445, + 3781, 3804, 2789, 0, 2482, 0, 1008, 2787, 2502, 3828, + 3851, 2744, 0, 2509, 0, 2516, 0, 1153, 2741, 2523, + 3875, 3898, 2740, 0, 0, 2923, 0, 2930, 0, 1300, + 2718, 3067, 3922, 3945, 2717, 0, 0, 1198, 3074, 2724, + 2399, 2417, 2721, 2701, 2453, 2454, 3969, 2670, 3993, 1285, + 2649, 2655, 2401, 2438, 3083, 2398, 1558, 2457, 2940, 1708, + 3084, 1557, 1385, 1709, 1508, 1754, 1710, 1755, 1778, 3120, + 1779, 3118, 1801, 1803, 1825, 4035, 4059, 4068, 1826, 2646, + 2643, 4086, 1287, 2624, 4127, 1288, 2600, 4168, 1315, 2599, + + 4209, 1345, 2597, 2596, 2595, 4249, 4273, 1427, 2594, 2635, + 2682, 4314, 2572, 4338, 1429, 2571, 2578, 2938, 2942, 2577, + 2576, 3096, 3142, 4379, 2566, 4403, 1489, 2540, 2544, 1600, + 3156, 2542, 3151, 3164, 2541, 2540, 3183, 3184, 4444, 2531, + 4468, 1603, 2468, 2469, 0, 1605, 3255, 2438, 3203, 3205, + 2372, 2371, 3249, 3268, 4509, 2343, 4533, 1606, 2341, 2348, + 0, 3320, 0, 3327, 0, 1396, 2306, 3385, 4574, 4597, + 2299, 0, 3095, 3096, 3178, 1827, 3122, 2399, 1848, 3420, + 2959, 1849, 1850, 1872, 3248, 3380, 1874, 1896, 3490, 2014, + 3492, 1959, 2016, 3337, 4622, 4646, 4655, 2275, 4672, 4696, + + 4719, 2263, 4743, 4766, 2239, 4790, 4813, 2234, 4837, 4860, + 2198, 2194, 4884, 1630, 2192, 2145, 3523, 4925, 2144, 2107, + 0, 3530, 0, 1615, 2106, 4016, 4949, 2103, 2060, 0, + 4023, 0, 4075, 0, 1639, 2059, 4109, 4973, 2058, 2055, + 0, 0, 4116, 0, 4150, 0, 1669, 2000, 4157, 4997, + 1999, 1976, 0, 0, 4191, 0, 4198, 0, 1748, 1951, + 4232, 5021, 1945, 1913, 0, 0, 1660, 4239, 1917, 3335, + 3399, 1914, 1897, 3400, 3420, 5045, 1882, 5069, 1661, 1842, + 1846, 2102, 2177, 2200, 2289, 2103, 2612, 4283, 2226, 2473, + 4266, 2680, 4348, 4267, 4331, 3249, 3271, 4289, 4332, 3399, + + 2610, 2611, 4396, 5111, 1815, 5128, 5152, 1662, 1814, 5193, + 1689, 1811, 5234, 1736, 1795, 5275, 1879, 1791, 5316, 1880, + 1747, 1744, 4365, 5357, 1741, 1698, 0, 1697, 3634, 3681, + 5381, 1670, 1664, 1671, 3728, 4289, 1616, 1580, 4374, 4418, + 5405, 1568, 1565, 1552, 1938, 4432, 1527, 4441, 4478, 1508, + 1506, 4479, 4483, 5429, 1494, 1454, 1442, 0, 1940, 4497, + 1438, 4506, 4543, 1437, 1400, 4544, 4548, 5453, 1390, 1389, + 1363, 0, 1941, 4562, 1362, 4571, 4591, 83, 110, 4666, + 4667, 5477, 195, 198, 225, 0, 4907, 0, 4914, 0, + 2006, 307, 5092, 5501, 0, 348, 0, 2474, 2635, 2657, + + 2659, 2681, 3003, 5107, 5162, 4670, 2703, 4291, 4356, 2682, + 4376, 4923, 4443, 2728, 2729, 5108, 5163, 5525, 1943, 0, + 372, 5566, 0, 399, 5590, 0, 418, 5614, 0, 480, + 5638, 0, 520, 5662, 0, 544, 4671, 4713, 5686, 551, + 580, 582, 610, 604, 5180, 0, 2061, 624, 643, 5216, + 0, 5223, 0, 2062, 696, 697, 0, 5257, 0, 5264, + 0, 2109, 698, 720, 0, 5298, 0, 5305, 0, 2142, + 721, 749, 0, 5339, 0, 5346, 0, 2193, 750, 752, + 0, 1995, 5548, 762, 4760, 4807, 763, 798, 4854, 5086, + 5710, 793, 0, 804, 4508, 2751, 2752, 2774, 2775, 5146, + + 2776, 2799, 2822, 2823, 5720, 2845, 5703, 5722, 2846, 3270, + 3489, 5755, 820, 864, 0, 0, 0, 0, 0, 0, + 5555, 0, 2282, 880, 887, 0, 5127, 5354, 897, 930, + 0, 1997, 5739, 931, 5563, 5724, 934, 976, 0, 0, + 2024, 5778, 985, 5752, 5772, 1012, 1039, 0, 0, 2151, + 5792, 1063, 5787, 5788, 1102, 1103, 0, 0, 2181, 5802, + 1107, 5800, 5810, 1132, 1160, 0, 0, 5814, 0, 5828, + 0, 2347, 1176, 0, 5085, 3553, 2868, 2870, 3103, 5824, + 5827, 5837, 5838, 5839, 5826, 3379, 5843, 2892, 4573, 5736, + 2981, 0, 7856, 0, 0, 0, 0, 0, 0, 5845, + + 5847, 1190, 1235, 0, 7856, 5871, 0, 7856, 0, 5887, + 0, 7856, 0, 5894, 0, 7856, 0, 5901, 0, 7856, + 0, 5908, 0, 7856, 0, 2270, 5915, 1236, 5923, 5924, + 1237, 2414, 0, 2980, 5086, 2982, 3004, 5926, 5356, 5565, + 3201, 5849, 4287, 3005, 5927, 5106, 5850, 3006, 5928, 0, + 5927, 0, 2272, 5954, 1240, 0, 2273, 5964, 1264, 0, + 2338, 5971, 1306, 0, 2396, 5978, 1322, 0, 2531, 5985, + 1356, 0, 5992, 0, 7856, 3177, 6001, 3442, 3444, 3467, + 6002, 6004, 6005, 6008, 3491, 6006, 3378, 6007, 2532, 6007, + 1357, 0, 0, 0, 0, 0, 0, 2533, 6043, 1361, + + 6016, 6037, 5852, 6039, 3554, 6040, 3555, 3556, 6017, 6052, + 6056, 3419, 0, 0, 0, 0, 0, 0, 0, 3466, + 6059, 3611, 3658, 3705, 6060, 6062, 6063, 3468, 6065, 6066, + 0, 0, 3609, 3610, 3634, 3656, 6067, 3680, 6068, 3681, + 6073, 6082, 6092, 6095, 3727, 6101, 3728, 6103, 3749, 5102, + 3773, 3774, 3797, 6106, 6111, 6115, 3775, 6116, 6122, 3820, + 6123, 6126, 6128, 3822, 6114, 6142, 6148, 3844, 3845, 6152, + 6153, 6154, 6155, 3869, 3890, 6158, 3891, 6162, 3914, 6163, + 6167, 3915, 3916, 3938, 6169, 6177, 4737, 6191, 3939, 6182, + 6194, 6197, 6198, 3961, 6204, 6203, 6205, 3963, 6206, 3985, + + 4084, 6213, 3987, 6218, 6227, 4125, 4613, 6219, 4308, 4032, + 6228, 4033, 4615, 4085, 6238, 6229, 6240, 4784, 4166, 6239, + 4207, 6241, 6268, 4208, 4665, 4666, 4669, 7856, 6296, 6304, + 6308, 6311, 6314, 6317, 6320, 6323, 6326, 6329, 6332, 6335, + 6338, 6341, 6344, 6347, 6350, 6353, 6356, 6360, 6364, 6367, + 6370, 6373, 6376, 6379, 6382, 6385, 6388, 6392, 6396, 6399, + 6402, 6406, 6408, 6411, 6414, 6417, 6420, 6423, 6426, 6429, + 6432, 6436, 6438, 6441, 6445, 6450, 6454, 6457, 6461, 6464, + 6467, 6470, 6473, 6476, 6479, 6482, 6486, 6490, 6493, 6497, + 6501, 6506, 6510, 6512, 6516, 6519, 6523, 6526, 6529, 6533, + + 6535, 6538, 6541, 6544, 6547, 6550, 6553, 6556, 6559, 6562, + 6566, 6568, 6571, 6574, 6577, 6581, 6583, 6586, 6589, 6594, + 6598, 6603, 6607, 6609, 6613, 6616, 6620, 6625, 6629, 6632, + 6635, 6638, 6641, 6644, 6647, 6650, 6654, 6658, 6661, 6665, + 6669, 6674, 6678, 6680, 6684, 6687, 6691, 6694, 6699, 6703, + 6708, 6712, 6714, 6718, 6721, 6725, 6728, 6731, 6734, 6738, + 6740, 6743, 6748, 6752, 6755, 6758, 6761, 6764, 6767, 6770, + 6773, 6776, 6780, 6782, 6785, 6788, 6791, 6795, 6797, 6800, + 6803, 6806, 6809, 6813, 6815, 6818, 6821, 6824, 6829, 6833, + 6838, 6842, 6844, 6848, 6851, 6855, 6860, 6864, 6867, 6870, + + 6873, 6876, 6879, 6882, 6885, 6889, 6893, 6896, 6900, 6904, + 6909, 6913, 6915, 6919, 6922, 6926, 6929, 6934, 6938, 6943, + 6947, 6949, 6953, 6956, 6960, 6963, 6966, 6971, 6975, 6980, + 6984, 6986, 6990, 6993, 6997, 7000, 7003, 7006, 7010, 7012, + 7015, 7020, 7024, 7027, 7030, 7033, 7036, 7039, 7042, 7045, + 7048, 7051, 7054, 7057, 7061, 7063, 7066, 7069, 7072, 7075, + 7079, 7081, 7084, 7087, 7090, 7093, 7096, 7100, 7102, 7105, + 7108, 7111, 7114, 7117, 7121, 7123, 7126, 7129, 7132, 7135, + 7140, 7144, 7149, 7153, 7155, 7159, 7162, 7166, 7171, 7175, + 7178, 7181, 7184, 7187, 7190, 7193, 7196, 7199, 7202, 7206, + + 7210, 7213, 7217, 7221, 7226, 7230, 7232, 7236, 7239, 7243, + 7246, 7251, 7255, 7260, 7264, 7266, 7270, 7273, 7277, 7280, + 7283, 7288, 7292, 7297, 7301, 7303, 7307, 7310, 7314, 7317, + 7320, 7325, 7329, 7334, 7338, 7340, 7344, 7347, 7351, 7354, + 7357, 7360, 7364, 7366, 7369, 7372, 7377, 7381, 7384, 7387, + 7390, 7393, 7396, 7399, 7402, 7405, 7408, 7411, 7414, 7418, + 7422, 7425, 7428, 7432, 7435, 7438, 7442, 7444, 7447, 7450, + 7454, 7456, 7459, 7462, 7465, 7469, 7471, 7474, 7477, 7480, + 7484, 7486, 7489, 7492, 7495, 7499, 7501, 7504, 7507, 7512, + 7516, 7521, 7525, 7527, 7531, 7534, 7538, 7543, 7547, 7550, + + 7553, 7556, 7559, 7562, 7565, 7568, 7571, 7575, 7577, 7580, + 7584, 7589, 7593, 7594, 7597, 7602, 7606, 7611, 7615, 7616, + 7619, 7622, 7627, 7631, 7636, 7640, 7641, 7644, 7647, 7652, + 7656, 7661, 7665, 7666, 7669, 7672, 7677, 7681, 7686, 7690, + 7691, 7694, 7697, 7700, 7704, 7706, 7711, 7715, 7718, 7721, + 7724, 7727, 7730, 7733, 7737, 7742, 7746, 7747, 7750, 7753, + 7756, 7759, 7762, 7765, 7768, 7771, 7774, 7777, 7782, 7786, + 7789, 7792, 7795, 7799, 7803, 7807, 7811, 7815, 7818, 7821, + 7825, 7828, 7831, 7834, 7837, 7840, 7844, 7847 } ; -static yyconst flex_int16_t yy_def[1930] = +static yyconst flex_int16_t yy_def[2189] = { 0, - 1469, 1, 1469, 1469, 1469, 1469, 1469, 1469, 1470, 1469, - 1469, 1469, 1469, 1469, 14, 1469, 1469, 1469, 1469, 14, - 20, 1471, 20, 20, 20, 20, 20, 20, 21, 21, + 1728, 1, 1728, 1728, 1728, 1728, 1728, 1728, 1729, 1728, + 1728, 1728, 1728, 1728, 14, 1728, 1728, 1728, 1728, 14, + 20, 1730, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - 21, 21, 1469, 1469, 1469, 1472, 1469, 21, 21, 20, - 1473, 50, 21, 21, 21, 1469, 1469, 1469, 1469, 1469, - 1469, 49, 1471, 1471, 52, 52, 52, 21, 21, 21, + 21, 21, 1728, 1728, 1728, 1731, 1728, 21, 21, 20, + 1732, 50, 21, 21, 21, 1728, 1728, 1728, 1728, 1728, + 1728, 49, 1730, 1730, 52, 52, 52, 21, 21, 21, 21, 52, 21, 21, 52, 21, 21, 21, 52, 21, 21, 21, 21, 21, 52, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, @@ -819,8 +898,8 @@ static yyconst flex_int16_t yy_def[1930] = 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - 1469, 1469, 21, 21, 154, 21, 21, 157, 1474, 1469, - 54, 1469, 162, 1475, 21, 21, 158, 21, 21, 21, + 1728, 1728, 21, 21, 154, 21, 21, 157, 1733, 1728, + 54, 1728, 162, 1734, 21, 21, 158, 21, 21, 21, 158, 21, 21, 21, 21, 21, 21, 158, 21, 21, 21, 21, 21, 21, 21, 158, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, @@ -831,9 +910,9 @@ static yyconst flex_int16_t yy_def[1930] = 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - 21, 21, 21, 21, 21, 262, 263, 158, 1476, 268, - 1477, 1478, 1469, 273, 1479, 1480, 1469, 1469, 1469, 1481, - 1482, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 262, 263, 158, 1735, 268, + 1736, 1737, 1728, 273, 1738, 1739, 1728, 1728, 1728, 1740, + 1741, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, @@ -841,9 +920,9 @@ static yyconst flex_int16_t yy_def[1930] = 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - 350, 21, 263, 265, 263, 265, 265, 357, 1483, 1469, - 356, 1484, 1485, 1469, 1469, 1469, 1469, 1486, 1487, 1488, - 1489, 1489, 1469, 1490, 1469, 375, 1491, 1482, 21, 21, + 350, 21, 263, 265, 263, 265, 265, 357, 1742, 1728, + 356, 1743, 1744, 1728, 1728, 1728, 1728, 1745, 1746, 1747, + 1748, 1748, 1728, 1749, 1728, 375, 1750, 1741, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, @@ -851,173 +930,201 @@ static yyconst flex_int16_t yy_def[1930] = 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 432, 433, 433, 438, 432, - 357, 441, 1492, 1493, 1469, 445, 1494, 1469, 1495, 1496, - 1469, 451, 1497, 1498, 1499, 1499, 1469, 1500, 1469, 459, - 1501, 1487, 1469, 1469, 1502, 1503, 1469, 1469, 1469, 1469, - 1504, 1505, 21, 21, 21, 21, 21, 21, 21, 21, + 357, 441, 1751, 1752, 1728, 445, 1753, 1728, 1754, 1755, + 1728, 451, 1756, 1757, 1758, 1758, 1728, 1759, 1728, 459, + 1760, 1746, 1728, 1728, 1761, 1762, 1728, 1728, 1728, 1728, + 1763, 1764, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - 21, 21, 21, 21, 21, 21, 21, 21, 21, 519, - 21, 433, 435, 433, 433, 525, 441, 527, 1506, 1469, - 1469, 1469, 1507, 1508, 1509, 1469, 1469, 1469, 1469, 1510, - 1511, 1469, 1512, 1513, 1469, 1469, 1469, 1469, 1514, 1515, - 1516, 1516, 1502, 1503, 1517, 1517, 1469, 1518, 1469, 559, - 1519, 1520, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 522, 21, 433, 435, 433, 433, 528, 441, + 530, 1765, 1728, 1728, 1728, 1766, 1767, 1768, 1728, 1728, + 1728, 1728, 1769, 1770, 1728, 1771, 1772, 1728, 1728, 1728, + 1728, 1773, 1774, 1775, 1775, 1761, 1762, 1776, 1776, 1728, + 1777, 1728, 562, 1778, 1779, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - 21, 21, 21, 21, 21, 21, 604, 604, 608, 527, - 610, 1521, 1522, 1469, 614, 1523, 1469, 617, 1524, 1469, - 1525, 1526, 1469, 623, 1527, 1528, 1528, 1469, 1529, 1469, - 630, 1530, 1531, 1532, 1532, 1533, 1534, 1535, 1535, 1469, - 1536, 1469, 642, 1537, 1538, 1469, 1539, 1469, 1540, 1541, - 1469, 1469, 1469, 1469, 1542, 1543, 611, 657, 657, 657, - 657, 657, 657, 657, 657, 657, 657, 657, 657, 657, - 657, 657, 657, 657, 657, 657, 657, 657, 657, 657, - 657, 657, 682, 682, 682, 657, 682, 687, 1544, 1469, - 1469, 1469, 1545, 1469, 1469, 1546, 1547, 1548, 1469, 1469, - - 1469, 1469, 1549, 1550, 1469, 1551, 1552, 1469, 1469, 1469, - 1469, 1553, 1554, 1469, 1555, 1469, 1556, 1557, 1469, 1469, - 1469, 1469, 1558, 1559, 1560, 1469, 1561, 1562, 1562, 1563, - 1564, 1565, 1565, 1469, 1566, 1469, 736, 1567, 1568, 1569, - 1569, 1569, 1569, 1569, 1569, 1569, 1569, 1569, 1569, 1569, - 1569, 1569, 1569, 1569, 1569, 1569, 1569, 1569, 1569, 758, - 1569, 758, 762, 762, 764, 1570, 1571, 1469, 768, 1572, - 1469, 771, 1573, 1469, 774, 1574, 1469, 1575, 1576, 1469, - 780, 1577, 1578, 1578, 1469, 1579, 1469, 787, 1580, 1581, - 1582, 1582, 1583, 1584, 1585, 1585, 1469, 1586, 1469, 799, - - 1587, 1588, 1589, 1469, 1590, 1591, 1591, 1592, 1593, 1594, - 1594, 1469, 1595, 1469, 814, 1596, 1597, 1598, 1469, 1599, - 1469, 1600, 1601, 1469, 1469, 1469, 1469, 1602, 1603, 1604, - 1604, 1604, 1604, 1604, 1604, 1604, 1604, 1604, 1604, 1604, - 1604, 1604, 1604, 843, 843, 845, 843, 843, 848, 1605, - 1469, 1469, 1469, 1606, 1469, 1469, 1607, 1469, 1469, 1608, - 1609, 1610, 1469, 1469, 1469, 1469, 1611, 1612, 1469, 1613, - 1614, 1469, 1469, 1469, 1469, 1615, 1616, 1469, 1617, 1469, - 1618, 1619, 1469, 1469, 1469, 1469, 1620, 1621, 1622, 1469, - 1623, 1469, 1624, 1625, 1469, 1469, 1469, 1469, 1626, 1627, - - 1628, 1629, 1469, 1630, 1631, 1631, 1632, 1633, 1634, 1634, - 1469, 1635, 1469, 913, 1636, 1637, 1638, 1638, 1638, 1638, - 1638, 1638, 1638, 1638, 1638, 1638, 1638, 1638, 1638, 1638, - 930, 1638, 1638, 1639, 1640, 1469, 936, 1641, 1469, 939, - 1642, 1469, 942, 1643, 1469, 945, 1644, 1469, 1645, 1469, - 1469, 951, 1646, 1647, 1647, 1469, 1648, 1469, 958, 1649, - 1650, 1651, 1651, 1652, 1653, 1654, 1654, 1469, 1655, 1469, - 970, 1656, 1657, 1658, 1469, 1659, 1660, 1660, 1661, 1662, - 1663, 1663, 1469, 1664, 1469, 985, 1665, 1666, 1667, 1668, - 1469, 1669, 1670, 1670, 1671, 1672, 1673, 1673, 1469, 1674, - - 1469, 1001, 1675, 1676, 1677, 1469, 1678, 1469, 1679, 1680, - 1469, 1469, 1469, 1469, 1681, 1682, 1683, 1683, 1683, 1683, - 1683, 1683, 1683, 1683, 1683, 1683, 1683, 1683, 1683, 1029, - 1683, 1684, 1469, 1469, 1469, 1685, 1469, 1469, 1686, 1469, - 1469, 1687, 1469, 1469, 1688, 1689, 1469, 1047, 1690, 1691, - 1469, 1469, 1692, 1693, 1694, 1469, 1695, 1696, 1469, 1469, - 1469, 1697, 1698, 1699, 1469, 1700, 1469, 1701, 1702, 1469, - 1469, 1469, 1703, 1704, 1705, 1706, 1469, 1707, 1469, 1708, - 1709, 1469, 1469, 1469, 1710, 1711, 1712, 1713, 1469, 1714, - 1469, 1715, 1716, 1469, 1469, 1469, 1717, 1718, 1719, 1720, - - 1721, 1469, 1722, 1723, 1723, 1724, 1725, 1726, 1726, 1469, - 1727, 1469, 1112, 1728, 1729, 1730, 1730, 1730, 1730, 1730, - 1730, 1730, 1730, 1730, 1730, 1730, 1731, 1469, 1469, 1129, - 1732, 1469, 1132, 1733, 1469, 1135, 1734, 1469, 1138, 1735, - 1469, 1141, 1736, 1469, 1469, 1469, 1737, 1738, 1739, 1740, - 1741, 1741, 1469, 1742, 1743, 1744, 1745, 1745, 1746, 1747, - 1748, 1748, 1469, 1749, 1750, 1751, 1752, 1469, 1753, 1754, - 1754, 1755, 1756, 1757, 1757, 1469, 1758, 1759, 1760, 1761, - 1762, 1469, 1763, 1764, 1764, 1765, 1766, 1767, 1767, 1469, - 1768, 1769, 1770, 1771, 1772, 1469, 1773, 1774, 1774, 1775, - - 1776, 1777, 1777, 1469, 1778, 1779, 1780, 1781, 1469, 1782, - 1469, 1783, 1784, 1469, 1469, 1469, 1785, 1786, 1787, 1788, - 1788, 1788, 1788, 1788, 1788, 1788, 1788, 1469, 1228, 1789, - 1790, 1469, 1791, 1792, 1469, 1793, 1794, 1469, 1795, 1796, - 1469, 1797, 1798, 1469, 1799, 1800, 1801, 1801, 1469, 1802, - 1803, 1804, 1805, 1806, 1469, 1807, 1808, 1469, 1809, 1469, - 1810, 1469, 1811, 1812, 1469, 1813, 1814, 1469, 1815, 1469, - 1816, 1817, 1469, 1818, 1819, 1469, 1820, 1469, 1821, 1822, - 1469, 1823, 1824, 1469, 1825, 1469, 1826, 1827, 1469, 1828, - 1829, 1830, 1469, 1831, 1832, 1832, 1833, 1834, 1835, 1835, - - 1469, 1836, 1837, 1838, 1839, 1839, 1839, 1839, 1469, 1840, - 1841, 1842, 1843, 1844, 1845, 1846, 1847, 1469, 1848, 1849, - 1469, 1850, 1851, 1852, 1852, 1853, 1854, 1855, 1856, 1469, - 1857, 1858, 1858, 1859, 1860, 1861, 1862, 1863, 1469, 1864, - 1865, 1865, 1866, 1867, 1868, 1869, 1870, 1469, 1871, 1872, - 1872, 1873, 1874, 1875, 1876, 1877, 1469, 1878, 1879, 1879, - 1880, 1881, 1882, 1883, 1469, 1884, 1469, 1885, 1886, 1469, - 1887, 1888, 1888, 1888, 1888, 1889, 1469, 1890, 1891, 1892, - 1893, 1894, 1895, 1896, 1896, 1897, 1898, 1899, 1469, 1469, - 1900, 1469, 1901, 1469, 1902, 1469, 1903, 1469, 1904, 1469, - - 1905, 1469, 1906, 1469, 1907, 1469, 1908, 1469, 1883, 1909, - 1469, 1884, 1910, 1910, 1885, 1886, 1911, 1888, 1888, 1888, - 1912, 1469, 1913, 1914, 1469, 1900, 1901, 1915, 1469, 1902, - 1903, 1916, 1469, 1904, 1905, 1917, 1469, 1906, 1907, 1918, - 1469, 1908, 1919, 1469, 1920, 1469, 1888, 1921, 1469, 1913, - 1922, 1923, 1924, 1925, 1926, 1919, 1927, 1469, 1920, 1928, - 1922, 1923, 1924, 1925, 1926, 1929, 1928, 1929, 0, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469 + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 616, 616, + 620, 530, 622, 1780, 1781, 1728, 626, 1782, 1728, 629, + 1783, 1728, 1784, 1785, 1728, 635, 1786, 1787, 1787, 1728, + 1788, 1728, 642, 1789, 1790, 1791, 1791, 1792, 1793, 1794, + 1794, 1728, 1795, 1728, 654, 1796, 1797, 1728, 1798, 1728, + 1799, 1800, 1728, 1728, 1728, 1728, 1801, 1802, 623, 669, + 669, 669, 669, 669, 669, 669, 669, 669, 669, 669, + 669, 669, 669, 669, 669, 669, 669, 669, 669, 669, + 669, 669, 669, 669, 669, 669, 669, 669, 669, 669, + + 669, 669, 669, 669, 669, 669, 706, 706, 706, 669, + 706, 711, 1803, 1728, 1728, 1728, 1804, 1728, 1728, 1805, + 1806, 1807, 1728, 1728, 1728, 1728, 1808, 1809, 1728, 1810, + 1811, 1728, 1728, 1728, 1728, 1812, 1813, 1728, 1814, 1728, + 1815, 1816, 1728, 1728, 1728, 1728, 1817, 1818, 1819, 1728, + 1820, 1821, 1821, 1822, 1823, 1824, 1824, 1728, 1825, 1728, + 760, 1826, 1827, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + 1828, 1828, 1828, 1828, 1828, 794, 1828, 794, 798, 798, + + 800, 1829, 1830, 1728, 804, 1831, 1728, 807, 1832, 1728, + 810, 1833, 1728, 1834, 1835, 1728, 816, 1836, 1837, 1837, + 1728, 1838, 1728, 823, 1839, 1840, 1841, 1841, 1842, 1843, + 1844, 1844, 1728, 1845, 1728, 835, 1846, 1847, 1848, 1728, + 1849, 1850, 1850, 1851, 1852, 1853, 1853, 1728, 1854, 1728, + 850, 1855, 1856, 1857, 1728, 1858, 1728, 1859, 1860, 1728, + 1728, 1728, 1728, 1861, 1862, 1863, 1863, 1863, 1863, 1863, + 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, + 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, 1863, 889, + 889, 891, 889, 889, 894, 1864, 1728, 1728, 1728, 1865, + + 1728, 1728, 1866, 1728, 1728, 1867, 1868, 1869, 1728, 1728, + 1728, 1728, 1870, 1871, 1728, 1872, 1873, 1728, 1728, 1728, + 1728, 1874, 1875, 1728, 1876, 1728, 1877, 1878, 1728, 1728, + 1728, 1728, 1879, 1880, 1881, 1728, 1882, 1728, 1883, 1884, + 1728, 1728, 1728, 1728, 1885, 1886, 1887, 1888, 1728, 1889, + 1890, 1890, 1891, 1892, 1893, 1893, 1728, 1894, 1728, 959, + 1895, 1896, 1897, 1897, 1897, 1897, 1897, 1897, 1897, 1897, + 1897, 1897, 1897, 1897, 1897, 1897, 1897, 1897, 1897, 1897, + 1897, 1897, 1897, 1897, 1897, 1897, 986, 1897, 1897, 1898, + 1899, 1728, 992, 1900, 1728, 995, 1901, 1728, 998, 1902, + + 1728, 1001, 1903, 1728, 1904, 1728, 1728, 1007, 1905, 1906, + 1906, 1728, 1907, 1728, 1014, 1908, 1909, 1910, 1910, 1911, + 1912, 1913, 1913, 1728, 1914, 1728, 1026, 1915, 1916, 1917, + 1728, 1918, 1919, 1919, 1920, 1921, 1922, 1922, 1728, 1923, + 1728, 1041, 1924, 1925, 1926, 1927, 1728, 1928, 1929, 1929, + 1930, 1931, 1932, 1932, 1728, 1933, 1728, 1057, 1934, 1935, + 1936, 1728, 1937, 1728, 1938, 1939, 1728, 1728, 1728, 1728, + 1940, 1941, 1942, 1942, 1942, 1942, 1942, 1942, 1942, 1942, + 1942, 1942, 1942, 1942, 1942, 1942, 1942, 1942, 1942, 1942, + 1942, 1942, 1942, 1942, 1942, 1095, 1942, 1943, 1728, 1728, + + 1728, 1944, 1728, 1728, 1945, 1728, 1728, 1946, 1728, 1728, + 1947, 1948, 1728, 1113, 1949, 1950, 1728, 1728, 1951, 1952, + 1953, 1728, 1954, 1955, 1728, 1728, 1728, 1956, 1957, 1958, + 1728, 1959, 1728, 1960, 1961, 1728, 1728, 1728, 1962, 1963, + 1964, 1965, 1728, 1966, 1728, 1967, 1968, 1728, 1728, 1728, + 1969, 1970, 1971, 1972, 1728, 1973, 1728, 1974, 1975, 1728, + 1728, 1728, 1976, 1977, 1978, 1979, 1980, 1728, 1981, 1982, + 1982, 1983, 1984, 1985, 1985, 1728, 1986, 1728, 1178, 1987, + 1988, 1989, 1989, 1989, 1989, 1989, 1989, 1989, 1989, 1989, + 1989, 1989, 1989, 1989, 1989, 1989, 1989, 1989, 1989, 1989, + + 1989, 1989, 1989, 1989, 1990, 1728, 1728, 1207, 1991, 1728, + 1210, 1992, 1728, 1213, 1993, 1728, 1216, 1994, 1728, 1219, + 1995, 1728, 1728, 1728, 1996, 1997, 1998, 1999, 2000, 2000, + 1728, 2001, 2002, 2003, 2004, 2004, 2005, 2006, 2007, 2007, + 1728, 2008, 2009, 2010, 2011, 1728, 2012, 2013, 2013, 2014, + 2015, 2016, 2016, 1728, 2017, 2018, 2019, 2020, 2021, 1728, + 2022, 2023, 2023, 2024, 2025, 2026, 2026, 1728, 2027, 2028, + 2029, 2030, 2031, 1728, 2032, 2033, 2033, 2034, 2035, 2036, + 2036, 1728, 2037, 2038, 2039, 2040, 1728, 2041, 1728, 2042, + 2043, 1728, 1728, 1728, 2044, 2045, 2046, 2047, 2047, 2047, + + 2047, 2047, 2047, 2047, 2047, 2047, 2047, 2047, 2047, 2047, + 2047, 2047, 2047, 2047, 2047, 2047, 2047, 1728, 1318, 2048, + 2049, 1728, 2050, 2051, 1728, 2052, 2053, 1728, 2054, 2055, + 1728, 2056, 2057, 1728, 2058, 2059, 2060, 2060, 1728, 2061, + 2062, 2063, 2064, 2065, 1728, 2066, 2067, 1728, 2068, 1728, + 2069, 1728, 2070, 2071, 1728, 2072, 2073, 1728, 2074, 1728, + 2075, 2076, 1728, 2077, 2078, 1728, 2079, 1728, 2080, 2081, + 1728, 2082, 2083, 1728, 2084, 1728, 2085, 2086, 1728, 2087, + 2088, 2089, 1728, 2090, 2091, 2091, 2092, 2093, 2094, 2094, + 1728, 2095, 2096, 2097, 2098, 2098, 2098, 2098, 2098, 2098, + + 2098, 2098, 2098, 2098, 2098, 2098, 2098, 2098, 2098, 2098, + 2098, 1728, 2099, 2100, 2101, 2102, 2103, 2104, 2105, 2106, + 1728, 2107, 2108, 1728, 2109, 2110, 2111, 2111, 2112, 2113, + 2114, 2115, 1728, 2116, 2117, 2117, 2118, 2119, 2120, 2121, + 2122, 1728, 2123, 2124, 2124, 2125, 2126, 2127, 2128, 2129, + 1728, 2130, 2131, 2131, 2132, 2133, 2134, 2135, 2136, 1728, + 2137, 2138, 2138, 2139, 2140, 2141, 2142, 1728, 2143, 1728, + 2144, 2145, 1728, 2146, 2147, 2147, 2147, 2147, 2147, 2147, + 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, + 2147, 2148, 1728, 2149, 2150, 2151, 2152, 2153, 2154, 2155, + + 2155, 2156, 2157, 2158, 1728, 1728, 2159, 1728, 2160, 1728, + 2161, 1728, 2162, 1728, 2163, 1728, 2164, 1728, 2165, 1728, + 2166, 1728, 2167, 1728, 2142, 2168, 1728, 2143, 2169, 2169, + 2144, 2145, 2170, 2147, 2147, 2147, 2147, 2147, 2147, 2147, + 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2171, + 1728, 2172, 2173, 1728, 2159, 2160, 2174, 1728, 2161, 2162, + 2175, 1728, 2163, 2164, 2176, 1728, 2165, 2166, 2177, 1728, + 2167, 2178, 1728, 2179, 1728, 2147, 2147, 2147, 2147, 2147, + 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2180, 1728, + 2172, 2181, 2182, 2183, 2184, 2185, 2178, 2186, 1728, 2179, + + 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, + 2147, 2147, 2187, 2181, 2182, 2183, 2184, 2185, 2188, 2147, + 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, + 2187, 2188, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, + 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, + 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, + 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, + 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, + 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, + 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, + + 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, + 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, + 2147, 2147, 2147, 2147, 2147, 2147, 2147, 0, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728 } ; -static yyconst flex_int16_t yy_nxt[7486] = +static yyconst flex_int16_t yy_nxt[7910] = { 0, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 13, 14, 15, 15, 15, 15, 15, 15, 16, 17, @@ -1028,823 +1135,870 @@ static yyconst flex_int16_t yy_nxt[7486] = 44, 212, 44, 44, 44, 57, 58, 44, 248, 44, 44, 44, 60, 61, 72, 44, 83, 84, 249, 44, 53, 44, 44, 44, 44, 206, 44, 44, 73, 85, - 44, 117, 79, 1233, 86, 74, 80, 742, 195, 44, + 44, 117, 79, 1157, 86, 74, 80, 686, 195, 44, - 118, 196, 81, 53, 197, 82, 198, 44, 48, 49, + 118, 196, 81, 687, 197, 82, 198, 44, 48, 49, 50, 50, 50, 50, 50, 50, 50, 51, 207, 201, - 1236, 52, 53, 54, 202, 182, 55, 52, 52, 52, + 942, 52, 53, 54, 202, 182, 55, 52, 52, 52, 52, 52, 52, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 54, - 53, 44, 183, 75, 44, 1239, 44, 44, 184, 236, - 87, 1242, 213, 76, 88, 344, 77, 214, 63, 56, + 53, 44, 183, 75, 44, 766, 44, 44, 184, 236, + 87, 53, 213, 76, 88, 344, 77, 214, 63, 56, 78, 63, 237, 63, 63, 208, 89, 44, 62, 52, 52, 52, 52, 52, 52, 52, 63, 242, 65, 345, - 66, 67, 53, 744, 63, 68, 53, 342, 119, 343, + 66, 67, 53, 768, 63, 68, 53, 342, 119, 343, 69, 90, 209, 91, 70, 243, 71, 210, 92, 93, - 120, 94, 121, 1245, 122, 95, 1310, 123, 53, 55, - 53, 53, 53, 53, 53, 53, 53, 53, 1469, 104, - 105, 217, 53, 746, 218, 453, 219, 245, 53, 53, + 120, 94, 121, 1335, 122, 95, 453, 123, 53, 55, + 53, 53, 53, 53, 53, 53, 53, 53, 1728, 104, + 105, 217, 53, 770, 218, 1161, 219, 245, 53, 53, 53, 53, 53, 53, 96, 106, 246, 97, 98, 107, 99, 53, 100, 108, 101, 109, 102, 111, 124, 103, - 53, 112, 110, 125, 126, 129, 127, 128, 453, 130, - 113, 114, 1469, 134, 115, 324, 116, 131, 141, 135, + 53, 112, 110, 125, 126, 129, 127, 128, 799, 130, + 113, 114, 1728, 134, 115, 324, 116, 131, 141, 135, 142, 136, 132, 137, 133, 146, 53, 138, 325, 139, - 140, 53, 355, 147, 143, 53, 524, 148, 1469, 149, - - 44, 144, 749, 44, 53, 44, 44, 156, 156, 156, - 156, 156, 156, 156, 1469, 417, 63, 53, 44, 63, - 418, 63, 63, 497, 1145, 501, 44, 153, 153, 153, - 153, 153, 153, 153, 63, 453, 498, 53, 153, 502, - 1469, 750, 63, 53, 153, 153, 153, 153, 153, 153, - 154, 155, 155, 155, 155, 155, 155, 1469, 53, 1233, - 1469, 156, 380, 381, 382, 383, 453, 156, 156, 156, + 140, 53, 355, 147, 143, 53, 527, 148, 1728, 149, + + 44, 144, 490, 44, 53, 44, 44, 156, 156, 156, + 156, 156, 156, 156, 1728, 417, 63, 53, 44, 63, + 418, 63, 63, 491, 500, 1070, 44, 153, 153, 153, + 153, 153, 153, 153, 63, 53, 708, 501, 153, 492, + 1728, 775, 63, 53, 153, 153, 153, 153, 153, 153, + 154, 155, 155, 155, 155, 155, 155, 1728, 53, 53, + 1728, 156, 380, 381, 382, 383, 1295, 156, 156, 156, 156, 156, 156, 49, 157, 157, 157, 157, 157, 157, - 157, 277, 384, 1469, 53, 158, 1469, 1469, 1469, 279, - 1236, 158, 158, 158, 158, 158, 158, 62, 158, 158, + 157, 277, 384, 1728, 53, 158, 1728, 1728, 1728, 279, + 1413, 158, 158, 158, 158, 158, 158, 62, 158, 158, 158, 158, 158, 158, 158, 161, 161, 161, 161, 161, - 161, 161, 573, 1469, 1469, 437, 161, 380, 381, 382, - 383, 574, 161, 161, 161, 161, 161, 161, 44, 265, - 453, 44, 1239, 44, 44, 53, 684, 464, 464, 162, - 163, 163, 163, 163, 163, 163, 44, 53, 1469, 453, - 164, 465, 1469, 1242, 44, 265, 164, 164, 164, 164, - 164, 164, 260, 576, 261, 261, 261, 261, 261, 261, - 261, 365, 577, 53, 1469, 261, 1469, 465, 1469, 532, - 453, 261, 261, 261, 261, 261, 261, 262, 263, 263, - 263, 263, 263, 263, 263, 277, 468, 741, 53, 264, - - 1469, 265, 1469, 1469, 470, 264, 264, 264, 264, 264, + 161, 161, 576, 1728, 1728, 437, 161, 1323, 504, 765, + 53, 577, 161, 161, 161, 161, 161, 161, 44, 265, + 767, 44, 505, 44, 44, 53, 1326, 464, 464, 162, + 163, 163, 163, 163, 163, 163, 44, 53, 1728, 53, + 164, 465, 1728, 774, 44, 265, 164, 164, 164, 164, + 164, 164, 260, 579, 261, 261, 261, 261, 261, 261, + 261, 365, 580, 53, 1728, 261, 1728, 465, 1728, 535, + 781, 261, 261, 261, 261, 261, 261, 262, 263, 263, + 263, 263, 263, 263, 263, 277, 468, 53, 1329, 264, + + 1728, 265, 1728, 1728, 470, 264, 264, 264, 264, 264, 264, 352, 352, 352, 352, 352, 352, 352, 371, 372, - 372, 372, 372, 372, 372, 542, 1469, 265, 266, 264, - 264, 264, 264, 264, 264, 264, 1469, 1469, 743, 543, - 264, 1469, 606, 53, 53, 757, 264, 264, 264, 264, + 372, 372, 372, 372, 372, 545, 1728, 265, 266, 264, + 264, 264, 264, 264, 264, 264, 688, 1728, 1332, 546, + 264, 380, 381, 382, 383, 689, 264, 264, 264, 264, 264, 264, 267, 268, 268, 268, 268, 268, 268, 268, - 269, 763, 1469, 1469, 270, 543, 1245, 1469, 606, 453, + 269, 779, 1335, 1728, 270, 546, 1728, 1728, 53, 1413, 270, 270, 270, 270, 270, 270, 53, 270, 270, 270, 270, 270, 270, 270, 273, 274, 274, 274, 274, 274, - 274, 275, 537, 1209, 818, 276, 1469, 53, 751, 55, - - 692, 276, 276, 276, 276, 276, 276, 277, 278, 278, - 278, 278, 278, 278, 278, 279, 53, 55, 1211, 280, - 818, 281, 1469, 396, 55, 280, 280, 280, 280, 280, - 280, 397, 440, 440, 440, 440, 440, 440, 440, 1012, - 55, 542, 55, 607, 398, 55, 747, 281, 349, 349, - 349, 349, 349, 349, 349, 1469, 53, 435, 55, 349, - 399, 55, 1217, 55, 1215, 349, 349, 349, 349, 349, - 349, 350, 351, 351, 351, 351, 351, 351, 745, 55, - 831, 1469, 352, 435, 1469, 1469, 830, 53, 352, 352, + 274, 275, 1728, 1728, 773, 276, 53, 777, 453, 53, + + 453, 276, 276, 276, 276, 276, 276, 277, 278, 278, + 278, 278, 278, 278, 278, 279, 782, 53, 786, 280, + 1223, 281, 453, 396, 53, 280, 280, 280, 280, 280, + 280, 397, 440, 440, 440, 440, 440, 440, 440, 53, + 793, 545, 1323, 619, 398, 55, 771, 281, 349, 349, + 349, 349, 349, 349, 349, 1728, 53, 435, 618, 349, + 399, 453, 55, 55, 784, 349, 349, 349, 349, 349, + 349, 350, 351, 351, 351, 351, 351, 351, 690, 53, + 55, 1728, 352, 435, 618, 854, 1728, 691, 352, 352, 352, 352, 352, 352, 262, 353, 353, 353, 353, 353, - 353, 353, 354, 354, 354, 354, 354, 354, 354, 546, - 1469, 1469, 1377, 354, 1469, 55, 55, 695, 55, 354, + 353, 353, 354, 354, 354, 354, 354, 354, 354, 769, + 1728, 854, 1728, 354, 1326, 453, 1329, 55, 53, 354, 354, 354, 354, 354, 354, 356, 357, 357, 357, 357, - 357, 357, 357, 55, 55, 840, 55, 358, 836, 837, - 1469, 1469, 1469, 358, 358, 358, 358, 358, 358, 361, + 357, 357, 357, 540, 549, 55, 1728, 358, 453, 1332, + 872, 716, 719, 358, 358, 358, 358, 358, 358, 361, 358, 358, 358, 358, 358, 358, 358, 365, 366, 366, - 366, 366, 366, 366, 366, 367, 652, 1469, 1469, 368, - 1310, 369, 1310, 453, 654, 368, 368, 368, 368, 368, - 368, 455, 456, 456, 456, 456, 456, 456, 521, 521, - 521, 521, 521, 521, 521, 646, 55, 369, 277, 373, + 366, 366, 366, 366, 366, 367, 664, 453, 1335, 368, + 453, 369, 1287, 1289, 666, 368, 368, 368, 368, 368, + 368, 455, 456, 456, 456, 456, 456, 456, 524, 524, + 524, 524, 524, 524, 524, 658, 778, 369, 277, 373, - 373, 373, 373, 373, 373, 373, 279, 1255, 1051, 647, - 374, 1469, 55, 1260, 55, 752, 374, 374, 374, 374, + 373, 373, 373, 373, 373, 373, 279, 53, 1068, 659, + 374, 1295, 55, 55, 1293, 776, 374, 374, 374, 374, 374, 374, 375, 376, 376, 376, 376, 376, 376, 53, - 55, 832, 753, 377, 646, 647, 1262, 1469, 1060, 377, - 377, 377, 377, 377, 377, 421, 53, 422, 1469, 423, - 424, 551, 552, 552, 552, 552, 552, 552, 55, 425, - 1268, 426, 427, 648, 428, 430, 648, 431, 431, 431, - 431, 431, 431, 431, 1469, 889, 55, 649, 431, 838, - 1469, 758, 55, 760, 431, 431, 431, 431, 431, 431, - 432, 433, 433, 433, 433, 433, 433, 433, 1270, 55, - - 55, 889, 434, 649, 435, 1071, 1469, 1276, 434, 434, - 434, 434, 434, 434, 555, 556, 556, 556, 556, 556, - 556, 626, 627, 627, 627, 627, 627, 627, 705, 756, + 55, 55, 780, 377, 658, 659, 879, 866, 1493, 377, + 377, 377, 377, 377, 377, 421, 53, 422, 1728, 423, + 424, 554, 555, 555, 555, 555, 555, 555, 55, 425, + 785, 426, 427, 660, 428, 430, 660, 431, 431, 431, + 431, 431, 431, 431, 1728, 53, 55, 661, 431, 873, + 1728, 55, 1413, 55, 431, 431, 431, 431, 431, 431, + 432, 433, 433, 433, 433, 433, 433, 433, 1413, 55, + + 876, 55, 434, 661, 435, 453, 1728, 1345, 434, 434, + 434, 434, 434, 434, 558, 559, 559, 559, 559, 559, + 559, 638, 639, 639, 639, 639, 639, 639, 729, 783, 435, 436, 434, 434, 434, 434, 434, 434, 434, 53, - 1278, 1083, 706, 434, 1469, 55, 55, 55, 705, 434, + 1117, 1350, 730, 434, 1352, 55, 55, 55, 729, 434, 434, 434, 434, 434, 434, 438, 439, 439, 439, 439, - 439, 439, 1469, 55, 55, 55, 440, 841, 706, 55, - 1469, 55, 440, 440, 440, 440, 440, 440, 267, 441, - 441, 441, 441, 441, 441, 441, 1284, 55, 1469, 55, - 442, 53, 917, 1005, 918, 1286, 442, 442, 442, 442, + 439, 439, 1728, 55, 55, 55, 440, 874, 730, 55, + 883, 55, 440, 440, 440, 440, 440, 440, 267, 441, + 441, 441, 441, 441, 441, 441, 1126, 55, 1728, 55, + 442, 1728, 884, 1728, 1728, 1358, 442, 442, 442, 442, 442, 442, 53, 442, 442, 442, 442, 442, 442, 442, - 445, 446, 446, 446, 446, 446, 446, 53, 700, 1005, - 1095, 447, 1217, 55, 55, 55, 853, 447, 447, 447, + 445, 446, 446, 446, 446, 446, 446, 1728, 724, 1728, + 1728, 447, 1360, 55, 55, 794, 899, 447, 447, 447, 447, 447, 447, 451, 452, 452, 452, 452, 452, 452, - 453, 55, 55, 55, 454, 1318, 919, 921, 923, 55, + 453, 55, 55, 55, 454, 887, 1728, 55, 796, 1137, 454, 454, 454, 454, 454, 454, 365, 457, 457, 457, - 457, 457, 457, 457, 367, 709, 1145, 55, 458, 1469, - 1365, 55, 924, 856, 458, 458, 458, 458, 458, 458, - 459, 460, 460, 460, 460, 460, 460, 453, 720, 55, - 1469, 461, 926, 1367, 55, 1469, 859, 461, 461, 461, + 457, 457, 457, 457, 367, 55, 1728, 935, 458, 1728, + 55, 55, 1728, 1366, 458, 458, 458, 458, 458, 458, + 459, 460, 460, 460, 460, 460, 460, 453, 55, 55, + 53, 461, 1728, 935, 963, 1728, 55, 461, 461, 461, 461, 461, 461, 277, 466, 466, 466, 466, 466, 466, - 466, 279, 55, 1390, 1394, 467, 1469, 1026, 55, 55, + 466, 279, 1368, 1149, 55, 467, 53, 1374, 55, 964, 55, 467, 467, 467, 467, 467, 467, 468, 469, 469, - 469, 469, 469, 469, 469, 470, 55, 55, 55, 471, - 1022, 472, 1398, 1469, 1028, 471, 471, 471, 471, 471, - 471, 634, 635, 635, 635, 635, 635, 635, 638, 639, - 639, 639, 639, 639, 639, 1402, 1406, 472, 482, 1469, - 1469, 1422, 483, 1469, 55, 484, 55, 714, 485, 714, - 486, 487, 488, 489, 518, 518, 518, 518, 518, 518, - 518, 715, 55, 1469, 55, 518, 1469, 55, 55, 1469, - - 755, 518, 518, 518, 518, 518, 518, 519, 520, 520, - 520, 520, 520, 520, 53, 55, 55, 715, 521, 1469, - 1469, 1469, 1469, 1444, 521, 521, 521, 521, 521, 521, - 432, 522, 522, 522, 522, 522, 522, 522, 523, 523, - 523, 523, 523, 523, 523, 825, 1469, 1469, 1469, 523, - 1076, 1469, 1088, 827, 1310, 523, 523, 523, 523, 523, - 523, 437, 525, 525, 525, 525, 525, 525, 525, 1217, - 1014, 716, 716, 526, 1095, 265, 1076, 1469, 1088, 526, - 526, 526, 526, 526, 526, 717, 1469, 725, 726, 726, - 726, 726, 726, 726, 728, 729, 729, 729, 729, 729, - - 729, 265, 526, 526, 526, 526, 526, 526, 526, 1469, - 864, 717, 1469, 526, 55, 55, 55, 453, 1035, 526, - 526, 526, 526, 526, 526, 527, 527, 527, 527, 527, - 527, 527, 55, 55, 55, 1469, 528, 1121, 1124, 1469, - 1208, 1469, 528, 528, 528, 528, 528, 528, 53, 528, - 528, 528, 528, 528, 528, 528, 365, 531, 531, 531, - 531, 531, 531, 531, 532, 1469, 1208, 1469, 533, 1245, - 369, 896, 1091, 1089, 533, 533, 533, 533, 533, 533, - 732, 733, 733, 733, 733, 733, 733, 783, 784, 784, - 784, 784, 784, 784, 55, 55, 369, 537, 538, 538, - - 538, 538, 538, 538, 538, 539, 873, 1083, 453, 540, - 1242, 541, 55, 55, 1038, 540, 540, 540, 540, 540, - 540, 791, 792, 792, 792, 792, 792, 792, 795, 796, - 796, 796, 796, 796, 796, 819, 55, 541, 365, 544, - 544, 544, 544, 544, 544, 544, 532, 884, 1079, 820, - 545, 1469, 1077, 1469, 55, 55, 545, 545, 545, 545, - 545, 545, 546, 547, 547, 547, 547, 547, 547, 547, - 548, 833, 1071, 55, 549, 820, 550, 1469, 1125, 1469, - 549, 549, 549, 549, 549, 549, 803, 804, 804, 804, - 804, 804, 804, 806, 807, 807, 807, 807, 807, 807, - - 819, 55, 550, 468, 557, 557, 557, 557, 557, 557, - 557, 470, 453, 1469, 1469, 558, 1469, 1239, 873, 55, - 821, 558, 558, 558, 558, 558, 558, 559, 560, 560, - 560, 560, 560, 560, 822, 884, 835, 1067, 561, 1469, - 1469, 1065, 1469, 1041, 561, 561, 561, 561, 561, 561, - 601, 821, 602, 602, 602, 602, 602, 602, 602, 896, - 822, 1060, 1469, 602, 1267, 1469, 55, 1044, 453, 602, - 602, 602, 602, 602, 602, 603, 604, 604, 604, 604, - 604, 604, 604, 1236, 55, 864, 1223, 605, 1469, 606, - 1267, 1469, 1056, 605, 605, 605, 605, 605, 605, 810, - - 811, 811, 811, 811, 811, 811, 726, 726, 726, 726, - 726, 726, 726, 869, 55, 606, 603, 605, 605, 605, - 605, 605, 605, 605, 1275, 1051, 453, 870, 605, 1283, - 55, 1469, 55, 55, 605, 605, 605, 605, 605, 605, - 437, 608, 608, 608, 608, 608, 608, 608, 55, 922, - 1275, 55, 609, 870, 1233, 1283, 1226, 1469, 609, 609, - 609, 609, 609, 609, 53, 609, 609, 609, 609, 609, - 609, 609, 610, 610, 610, 610, 610, 610, 610, 1012, - 1051, 453, 1147, 611, 1364, 1393, 1397, 1014, 1233, 611, - 611, 611, 611, 611, 611, 53, 611, 611, 611, 611, - - 611, 611, 611, 614, 615, 615, 615, 615, 615, 615, - 1364, 1393, 1397, 1401, 616, 1405, 1443, 55, 55, 55, - 616, 616, 616, 616, 616, 616, 365, 457, 457, 457, - 457, 457, 457, 457, 532, 55, 55, 55, 458, 1401, - 55, 1405, 1443, 869, 458, 458, 458, 458, 458, 458, - 617, 618, 618, 618, 618, 618, 618, 1469, 55, 1220, - 1221, 619, 1252, 55, 1128, 1245, 1242, 619, 619, 619, - 619, 619, 619, 623, 624, 624, 624, 624, 624, 624, - 453, 55, 1222, 1469, 625, 1239, 1236, 55, 55, 55, - 625, 625, 625, 625, 625, 625, 537, 628, 628, 628, - - 628, 628, 628, 628, 539, 55, 55, 55, 629, 1233, - 1128, 1305, 55, 1012, 629, 629, 629, 629, 629, 629, - 630, 631, 631, 631, 631, 631, 631, 453, 1224, 1217, - 55, 632, 1014, 1373, 825, 55, 55, 632, 632, 632, - 632, 632, 632, 546, 640, 640, 640, 640, 640, 640, - 640, 548, 1060, 55, 55, 641, 1374, 1008, 55, 55, - 1236, 641, 641, 641, 641, 641, 641, 642, 643, 643, - 643, 643, 643, 643, 453, 1071, 55, 55, 644, 1451, - 1006, 55, 1097, 1239, 644, 644, 644, 644, 644, 644, - 468, 650, 650, 650, 650, 650, 650, 650, 470, 55, - - 1375, 1452, 651, 1453, 1454, 1451, 55, 55, 651, 651, - 651, 651, 651, 651, 652, 653, 653, 653, 653, 653, - 653, 653, 654, 55, 55, 55, 655, 1452, 656, 1453, - 1454, 55, 655, 655, 655, 655, 655, 655, 842, 920, - 453, 55, 847, 847, 847, 847, 847, 847, 847, 55, - 878, 878, 55, 834, 656, 680, 680, 680, 680, 680, - 680, 680, 1455, 1083, 879, 1469, 680, 1460, 927, 55, - 55, 1242, 680, 680, 680, 680, 680, 680, 681, 681, - 681, 681, 681, 681, 681, 925, 1095, 55, 1455, 681, - 879, 1469, 1466, 1460, 1245, 681, 681, 681, 681, 681, - - 681, 603, 682, 682, 682, 682, 682, 682, 682, 683, - 683, 683, 683, 683, 683, 683, 1145, 1215, 1466, 1044, - 683, 1085, 55, 453, 1310, 1217, 683, 683, 683, 683, - 683, 683, 437, 685, 685, 685, 685, 685, 685, 685, - 55, 1041, 1073, 453, 686, 1038, 1062, 453, 1035, 1306, - 686, 686, 686, 686, 686, 686, 53, 686, 686, 686, - 686, 686, 686, 686, 687, 687, 687, 687, 687, 687, - 687, 1215, 1053, 453, 1147, 688, 453, 1128, 1044, 1469, - 1041, 688, 688, 688, 688, 688, 688, 53, 688, 688, - 688, 688, 688, 688, 688, 537, 691, 691, 691, 691, - - 691, 691, 691, 692, 55, 880, 880, 693, 1038, 541, - 1035, 1128, 1014, 693, 693, 693, 693, 693, 693, 881, - 1469, 827, 55, 804, 804, 804, 804, 804, 804, 804, - 896, 55, 1097, 890, 839, 541, 546, 694, 694, 694, - 694, 694, 694, 694, 695, 881, 1469, 891, 696, 55, - 550, 55, 890, 892, 696, 696, 696, 696, 696, 696, - 1044, 341, 720, 892, 890, 884, 1469, 893, 1085, 55, - 1023, 1041, 892, 891, 1006, 709, 550, 700, 701, 701, - 701, 701, 701, 701, 701, 702, 1469, 1024, 1007, 703, - 880, 704, 1469, 893, 1006, 703, 703, 703, 703, 703, - - 703, 902, 903, 903, 903, 903, 903, 903, 1469, 878, - 873, 1073, 1469, 1008, 1007, 1008, 1038, 704, 537, 707, - 707, 707, 707, 707, 707, 707, 692, 1009, 700, 1469, - 708, 869, 864, 55, 1469, 1062, 708, 708, 708, 708, - 708, 708, 709, 710, 710, 710, 710, 710, 710, 710, - 711, 55, 1035, 1009, 712, 1469, 713, 1053, 950, 1420, - 712, 712, 712, 712, 712, 712, 905, 906, 906, 906, - 906, 906, 906, 909, 910, 910, 910, 910, 910, 910, - 1056, 55, 713, 546, 718, 718, 718, 718, 718, 718, - 718, 695, 935, 1044, 1057, 719, 1041, 1038, 1035, 55, - - 1033, 719, 719, 719, 719, 719, 719, 720, 721, 721, - 721, 721, 721, 721, 721, 722, 1021, 935, 825, 723, - 1057, 724, 1014, 827, 652, 723, 723, 723, 723, 723, - 723, 954, 955, 955, 955, 955, 955, 955, 962, 963, - 963, 963, 963, 963, 963, 55, 55, 724, 652, 734, - 734, 734, 734, 734, 734, 734, 654, 821, 819, 898, - 735, 859, 886, 55, 55, 856, 735, 735, 735, 735, - 735, 735, 736, 737, 737, 737, 737, 737, 737, 1017, - 1027, 875, 853, 738, 866, 950, 935, 859, 1056, 738, - 738, 738, 738, 738, 738, 53, 53, 53, 53, 53, - - 53, 53, 1469, 856, 853, 935, 53, 55, 55, 55, - 827, 55, 53, 53, 53, 53, 53, 53, 966, 967, - 967, 967, 967, 967, 967, 55, 55, 55, 1469, 55, - 740, 758, 654, 759, 759, 759, 759, 759, 759, 759, - 1018, 1019, 1020, 1025, 759, 720, 898, 55, 1120, 55, - 759, 759, 759, 759, 759, 759, 974, 975, 975, 975, - 975, 975, 975, 1065, 1065, 55, 55, 55, 53, 760, - 761, 761, 761, 761, 761, 761, 761, 1066, 1469, 1119, - 859, 761, 1227, 546, 55, 716, 714, 761, 761, 761, - 761, 761, 761, 977, 978, 978, 978, 978, 978, 978, - - 709, 1372, 886, 1066, 1469, 53, 682, 682, 682, 682, - 682, 682, 682, 981, 982, 982, 982, 982, 982, 982, - 990, 991, 991, 991, 991, 991, 991, 993, 994, 994, - 994, 994, 994, 994, 997, 998, 998, 998, 998, 998, - 998, 53, 683, 683, 683, 683, 683, 683, 683, 856, - 537, 705, 700, 683, 875, 853, 866, 863, 779, 683, - 683, 683, 683, 683, 683, 762, 353, 353, 353, 353, - 353, 353, 353, 267, 764, 764, 764, 764, 764, 764, - 764, 767, 859, 856, 853, 765, 851, 767, 652, 827, - 654, 765, 765, 765, 765, 765, 765, 53, 765, 765, - - 765, 765, 765, 765, 765, 768, 769, 769, 769, 769, - 769, 769, 468, 648, 646, 722, 770, 695, 711, 692, - 702, 779, 770, 770, 770, 770, 770, 770, 537, 628, - 628, 628, 628, 628, 628, 628, 692, 767, 695, 692, - 629, 767, 53, 53, 53, 1067, 629, 629, 629, 629, - 629, 629, 771, 772, 772, 772, 772, 772, 772, 1068, - 754, 53, 53, 773, 748, 654, 470, 546, 722, 773, - 773, 773, 773, 773, 773, 546, 640, 640, 640, 640, - 640, 640, 640, 695, 695, 1068, 365, 641, 542, 537, - 711, 692, 1067, 641, 641, 641, 641, 641, 641, 774, - - 775, 775, 775, 775, 775, 775, 1469, 702, 699, 622, - 776, 613, 695, 692, 690, 613, 776, 776, 776, 776, - 776, 776, 780, 781, 781, 781, 781, 781, 781, 453, - 603, 601, 1469, 782, 679, 678, 677, 676, 675, 782, - 782, 782, 782, 782, 782, 700, 785, 785, 785, 785, - 785, 785, 785, 702, 674, 673, 672, 786, 671, 670, - 669, 668, 667, 786, 786, 786, 786, 786, 786, 787, - 788, 788, 788, 788, 788, 788, 453, 666, 665, 664, - 789, 663, 662, 661, 660, 659, 789, 789, 789, 789, - 789, 789, 709, 797, 797, 797, 797, 797, 797, 797, - - 711, 658, 657, 468, 798, 654, 470, 464, 548, 532, - 798, 798, 798, 798, 798, 798, 799, 800, 800, 800, - 800, 800, 800, 453, 539, 622, 613, 801, 532, 613, - 607, 607, 600, 801, 801, 801, 801, 801, 801, 720, - 812, 812, 812, 812, 812, 812, 812, 722, 599, 598, - 597, 813, 596, 595, 594, 593, 592, 813, 813, 813, - 813, 813, 813, 814, 815, 815, 815, 815, 815, 815, - 453, 591, 590, 338, 816, 239, 589, 588, 587, 586, - 816, 816, 816, 816, 816, 816, 652, 823, 823, 823, - 823, 823, 823, 823, 654, 585, 584, 583, 824, 582, - - 581, 580, 579, 578, 824, 824, 824, 824, 824, 824, - 825, 826, 826, 826, 826, 826, 826, 826, 827, 575, - 572, 571, 828, 570, 829, 569, 568, 567, 828, 828, - 828, 828, 828, 828, 903, 903, 903, 903, 903, 903, - 903, 975, 975, 975, 975, 975, 975, 975, 1077, 1077, - 829, 55, 1079, 843, 843, 843, 843, 843, 843, 843, - 566, 565, 1078, 1469, 843, 564, 1080, 563, 55, 55, - 843, 843, 843, 843, 843, 843, 844, 844, 844, 844, - 844, 844, 844, 470, 279, 453, 55, 844, 1078, 1469, - 365, 548, 1080, 844, 844, 844, 844, 844, 844, 845, - - 846, 846, 846, 846, 846, 846, 1123, 532, 539, 453, - 847, 536, 55, 450, 444, 532, 847, 847, 847, 847, - 847, 847, 267, 848, 848, 848, 848, 848, 848, 848, - 55, 530, 444, 436, 849, 430, 517, 516, 515, 1419, - 849, 849, 849, 849, 849, 849, 53, 849, 849, 849, - 849, 849, 849, 849, 700, 852, 852, 852, 852, 852, - 852, 852, 853, 514, 1079, 513, 854, 512, 704, 511, - 510, 509, 854, 854, 854, 854, 854, 854, 1469, 991, - 991, 991, 991, 991, 991, 991, 508, 507, 506, 505, - 55, 1089, 1089, 504, 704, 709, 855, 855, 855, 855, - - 855, 855, 855, 856, 1469, 1090, 1469, 857, 55, 713, - 55, 1091, 1091, 857, 857, 857, 857, 857, 857, 1116, - 503, 500, 499, 496, 495, 1092, 1469, 494, 55, 493, - 492, 1090, 1469, 120, 491, 713, 720, 858, 858, 858, - 858, 858, 858, 858, 859, 490, 481, 1418, 860, 480, - 724, 1092, 1469, 479, 860, 860, 860, 860, 860, 860, - 1101, 1102, 1102, 1102, 1102, 1102, 1102, 1104, 1105, 1105, - 1105, 1105, 1105, 1105, 478, 477, 724, 864, 865, 865, - 865, 865, 865, 865, 865, 866, 476, 475, 474, 867, - 473, 868, 277, 470, 279, 867, 867, 867, 867, 867, - - 867, 1108, 1109, 1109, 1109, 1109, 1109, 1109, 1151, 1152, - 1152, 1152, 1152, 1152, 1152, 1209, 453, 868, 700, 871, - 871, 871, 871, 871, 871, 871, 853, 367, 450, 1210, - 872, 444, 444, 437, 437, 55, 872, 872, 872, 872, - 872, 872, 873, 874, 874, 874, 874, 874, 874, 874, - 875, 55, 55, 55, 876, 1210, 877, 429, 420, 419, - 876, 876, 876, 876, 876, 876, 416, 1122, 415, 55, - 55, 1157, 1158, 1158, 1158, 1158, 1158, 1158, 1209, 55, - 1117, 1118, 877, 709, 882, 882, 882, 882, 882, 882, - 882, 856, 1469, 414, 413, 883, 412, 55, 411, 410, - - 409, 883, 883, 883, 883, 883, 883, 884, 885, 885, - 885, 885, 885, 885, 885, 886, 1447, 408, 1469, 887, - 407, 888, 406, 405, 404, 887, 887, 887, 887, 887, - 887, 1161, 1162, 1162, 1162, 1162, 1162, 1162, 1167, 1168, - 1168, 1168, 1168, 1168, 1168, 1211, 403, 888, 720, 894, - 894, 894, 894, 894, 894, 894, 859, 402, 401, 1212, - 895, 400, 395, 394, 393, 392, 895, 895, 895, 895, - 895, 895, 896, 897, 897, 897, 897, 897, 897, 897, - 898, 391, 320, 390, 899, 1212, 900, 389, 388, 387, - 899, 899, 899, 899, 899, 899, 1170, 1171, 1171, 1171, - - 1171, 1171, 1171, 1174, 1175, 1175, 1175, 1175, 1175, 1175, - 1211, 386, 900, 825, 911, 911, 911, 911, 911, 911, - 911, 827, 385, 379, 1469, 912, 279, 367, 275, 364, - 1255, 912, 912, 912, 912, 912, 912, 913, 914, 914, - 914, 914, 914, 914, 1256, 272, 360, 266, 915, 260, - 1469, 143, 348, 347, 915, 915, 915, 915, 915, 915, - 55, 1255, 928, 928, 928, 928, 928, 928, 928, 346, - 1256, 341, 320, 928, 340, 1469, 339, 338, 55, 928, - 928, 928, 928, 928, 928, 929, 929, 929, 929, 929, - 929, 929, 337, 336, 335, 334, 929, 333, 332, 331, - - 330, 1469, 929, 929, 929, 929, 929, 929, 607, 930, - 930, 930, 930, 930, 930, 930, 329, 328, 327, 326, - 931, 323, 435, 322, 321, 320, 931, 931, 931, 931, - 931, 931, 1181, 1182, 1182, 1182, 1182, 1182, 1182, 1184, - 1185, 1185, 1185, 1185, 1185, 1185, 319, 1260, 435, 931, - 931, 931, 931, 931, 931, 931, 318, 317, 316, 315, - 931, 1261, 314, 313, 312, 311, 931, 931, 931, 931, - 931, 931, 267, 932, 932, 932, 932, 932, 932, 932, - 310, 309, 308, 307, 933, 306, 305, 1261, 304, 303, - 933, 933, 933, 933, 933, 933, 53, 933, 933, 933, - - 933, 933, 933, 933, 936, 937, 937, 937, 937, 937, - 937, 302, 301, 300, 299, 938, 298, 297, 143, 296, - 295, 938, 938, 938, 938, 938, 938, 700, 785, 785, - 785, 785, 785, 785, 785, 853, 294, 293, 292, 786, - 291, 290, 289, 288, 1260, 786, 786, 786, 786, 786, - 786, 939, 940, 940, 940, 940, 940, 940, 1469, 287, - 286, 285, 941, 284, 283, 282, 279, 267, 941, 941, - 941, 941, 941, 941, 709, 797, 797, 797, 797, 797, - 797, 797, 856, 272, 1469, 259, 798, 258, 257, 256, - 255, 1262, 798, 798, 798, 798, 798, 798, 942, 943, - - 943, 943, 943, 943, 943, 1263, 254, 253, 252, 944, - 251, 250, 247, 244, 241, 944, 944, 944, 944, 944, - 944, 720, 812, 812, 812, 812, 812, 812, 812, 859, - 240, 1263, 239, 813, 238, 235, 234, 233, 1262, 813, - 813, 813, 813, 813, 813, 945, 946, 946, 946, 946, - 946, 946, 1469, 232, 231, 230, 947, 229, 228, 227, - 226, 225, 947, 947, 947, 947, 947, 947, 951, 952, - 952, 952, 952, 952, 952, 453, 224, 223, 1469, 953, - 222, 221, 220, 216, 215, 953, 953, 953, 953, 953, - 953, 864, 956, 956, 956, 956, 956, 956, 956, 866, - - 205, 204, 203, 957, 200, 199, 194, 193, 192, 957, - 957, 957, 957, 957, 957, 958, 959, 959, 959, 959, - 959, 959, 453, 191, 190, 189, 960, 188, 187, 186, - 185, 181, 960, 960, 960, 960, 960, 960, 873, 968, - 968, 968, 968, 968, 968, 968, 875, 180, 179, 178, - 969, 177, 176, 175, 174, 173, 969, 969, 969, 969, - 969, 969, 970, 971, 971, 971, 971, 971, 971, 453, - 172, 171, 170, 972, 169, 168, 167, 166, 165, 972, - 972, 972, 972, 972, 972, 884, 983, 983, 983, 983, - 983, 983, 983, 886, 160, 151, 150, 984, 145, 59, - - 47, 45, 1469, 984, 984, 984, 984, 984, 984, 985, - 986, 986, 986, 986, 986, 986, 453, 1469, 1469, 1469, - 987, 1469, 1469, 1469, 1469, 1469, 987, 987, 987, 987, - 987, 987, 896, 999, 999, 999, 999, 999, 999, 999, - 898, 1469, 1469, 1469, 1000, 1469, 1469, 1469, 1469, 1469, - 1000, 1000, 1000, 1000, 1000, 1000, 1001, 1002, 1002, 1002, - 1002, 1002, 1002, 453, 1469, 1469, 1469, 1003, 1469, 1469, - 1469, 1469, 1469, 1003, 1003, 1003, 1003, 1003, 1003, 825, - 1010, 1010, 1010, 1010, 1010, 1010, 1010, 827, 1469, 1469, - 1469, 1011, 1469, 1469, 1469, 1469, 1469, 1011, 1011, 1011, - - 1011, 1011, 1011, 1012, 1013, 1013, 1013, 1013, 1013, 1013, - 1013, 1014, 1469, 1469, 1469, 1015, 1469, 1016, 1469, 1469, - 1469, 1015, 1015, 1015, 1015, 1015, 1015, 1188, 1189, 1189, - 1189, 1189, 1189, 1189, 1195, 1196, 1196, 1196, 1196, 1196, - 1196, 1469, 1469, 1016, 55, 607, 1029, 1029, 1029, 1029, - 1029, 1029, 1029, 1469, 1469, 1469, 1469, 1030, 1469, 1469, - 1469, 1469, 55, 1030, 1030, 1030, 1030, 1030, 1030, 53, - 1030, 1030, 1030, 1030, 1030, 1030, 1030, 55, 267, 1031, - 1031, 1031, 1031, 1031, 1031, 1031, 1198, 1199, 1199, 1199, - 1199, 1199, 1199, 1469, 1469, 55, 864, 1034, 1034, 1034, - - 1034, 1034, 1034, 1034, 1035, 1469, 1469, 1469, 1036, 1469, - 868, 1469, 1469, 1469, 1036, 1036, 1036, 1036, 1036, 1036, - 1202, 1203, 1203, 1203, 1203, 1203, 1203, 1102, 1102, 1102, - 1102, 1102, 1102, 1102, 55, 1469, 868, 873, 1037, 1037, - 1037, 1037, 1037, 1037, 1037, 1038, 1469, 1268, 1268, 1039, - 1469, 877, 55, 1469, 1469, 1039, 1039, 1039, 1039, 1039, - 1039, 1269, 1469, 1225, 1247, 1248, 1248, 1248, 1248, 1248, - 1248, 1469, 1469, 1469, 1469, 1270, 1469, 877, 884, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1041, 1269, 1469, 1271, - 1042, 1469, 888, 1469, 1469, 1270, 1042, 1042, 1042, 1042, - - 1042, 1042, 1168, 1168, 1168, 1168, 1168, 1168, 1168, 1469, - 1469, 1469, 1469, 1469, 1276, 1271, 1276, 1469, 888, 896, - 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1044, 1277, 1469, - 1469, 1045, 1469, 900, 1469, 1469, 1278, 1045, 1045, 1045, - 1045, 1045, 1045, 1182, 1182, 1182, 1182, 1182, 1182, 1182, - 1279, 1469, 1469, 1469, 1277, 1469, 1469, 1469, 1278, 900, - 1047, 1048, 1048, 1048, 1048, 1048, 1048, 1049, 1469, 1469, - 1469, 1050, 1469, 1469, 1469, 1469, 1279, 1050, 1050, 1050, - 1050, 1050, 1050, 1051, 1052, 1052, 1052, 1052, 1052, 1052, - 1052, 1053, 1469, 1469, 1469, 1054, 1469, 1055, 1469, 1469, - - 1284, 1054, 1054, 1054, 1054, 1054, 1054, 1196, 1196, 1196, - 1196, 1196, 1196, 1196, 1285, 1469, 1469, 1469, 1469, 1284, - 1469, 1286, 1469, 1055, 864, 1058, 1058, 1058, 1058, 1058, - 1058, 1058, 1035, 1469, 1469, 1287, 1059, 1469, 1469, 1469, - 1285, 1469, 1059, 1059, 1059, 1059, 1059, 1059, 1060, 1061, - 1061, 1061, 1061, 1061, 1061, 1061, 1062, 1469, 1286, 1469, - 1063, 1287, 1064, 1469, 1469, 1469, 1063, 1063, 1063, 1063, - 1063, 1063, 1469, 1292, 1293, 1293, 1293, 1293, 1293, 1293, - 1295, 1296, 1296, 1296, 1296, 1296, 1296, 1469, 1064, 873, - 1069, 1069, 1069, 1069, 1069, 1069, 1069, 1038, 1469, 1469, - - 1469, 1070, 1469, 1469, 1469, 1469, 1469, 1070, 1070, 1070, - 1070, 1070, 1070, 1071, 1072, 1072, 1072, 1072, 1072, 1072, - 1072, 1073, 1469, 1469, 1469, 1074, 1469, 1075, 1469, 1469, - 1318, 1074, 1074, 1074, 1074, 1074, 1074, 1299, 1300, 1300, - 1300, 1300, 1300, 1300, 1319, 1469, 1469, 1469, 1469, 1469, - 1469, 1318, 1469, 1075, 884, 1081, 1081, 1081, 1081, 1081, - 1081, 1081, 1041, 1469, 1469, 1469, 1082, 1469, 1469, 1469, - 1319, 55, 1082, 1082, 1082, 1082, 1082, 1082, 1083, 1084, - 1084, 1084, 1084, 1084, 1084, 1084, 1085, 55, 1469, 55, - 1086, 1469, 1087, 1469, 1469, 1469, 1086, 1086, 1086, 1086, - - 1086, 1086, 1307, 1469, 1469, 55, 1324, 1325, 1325, 1325, - 1325, 1325, 1325, 1469, 1365, 1469, 1308, 1469, 1087, 896, - 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1044, 1366, 1469, - 1469, 1094, 1469, 1469, 1469, 1469, 1469, 1094, 1094, 1094, - 1094, 1094, 1094, 1095, 1096, 1096, 1096, 1096, 1096, 1096, - 1096, 1097, 1469, 1469, 1366, 1098, 1469, 1099, 1469, 1469, - 1469, 1098, 1098, 1098, 1098, 1098, 1098, 1329, 1330, 1330, - 1330, 1330, 1330, 1330, 1332, 1333, 1333, 1333, 1333, 1333, - 1333, 1365, 1469, 1099, 1012, 1110, 1110, 1110, 1110, 1110, - 1110, 1110, 1014, 1469, 1469, 1469, 1111, 1469, 1469, 1469, - - 1469, 1367, 1111, 1111, 1111, 1111, 1111, 1111, 1112, 1113, - 1113, 1113, 1113, 1113, 1113, 1368, 1469, 1469, 1469, 1114, - 1469, 1469, 1469, 1469, 1469, 1114, 1114, 1114, 1114, 1114, - 1114, 55, 607, 1126, 1126, 1126, 1126, 1126, 1126, 1126, - 1469, 1368, 1469, 1469, 933, 1469, 1469, 1469, 1469, 55, - 933, 933, 933, 933, 933, 933, 53, 933, 933, 933, - 933, 933, 933, 933, 55, 267, 1031, 1031, 1031, 1031, - 1031, 1031, 1031, 1338, 1339, 1339, 1339, 1339, 1339, 1339, - 1469, 1367, 55, 1129, 1130, 1130, 1130, 1130, 1130, 1130, - 1469, 1469, 1469, 1469, 1131, 1469, 1469, 1469, 1469, 1469, - - 1131, 1131, 1131, 1131, 1131, 1131, 864, 956, 956, 956, - 956, 956, 956, 956, 1035, 1469, 1469, 1469, 957, 1469, - 1469, 1469, 1469, 1390, 957, 957, 957, 957, 957, 957, - 1132, 1133, 1133, 1133, 1133, 1133, 1133, 1391, 1469, 1469, - 1469, 1134, 1469, 1469, 1469, 1469, 1469, 1134, 1134, 1134, - 1134, 1134, 1134, 873, 968, 968, 968, 968, 968, 968, - 968, 1038, 1469, 1391, 1469, 969, 1469, 1469, 1469, 1469, - 1390, 969, 969, 969, 969, 969, 969, 1135, 1136, 1136, - 1136, 1136, 1136, 1136, 1469, 1469, 1469, 1469, 1137, 1469, - 1469, 1469, 1469, 1469, 1137, 1137, 1137, 1137, 1137, 1137, - - 884, 983, 983, 983, 983, 983, 983, 983, 1041, 1469, - 1469, 1469, 984, 1469, 1469, 1469, 1469, 1394, 984, 984, - 984, 984, 984, 984, 1138, 1139, 1139, 1139, 1139, 1139, - 1139, 1395, 1469, 1469, 1469, 1140, 1469, 1469, 1469, 1469, - 1469, 1140, 1140, 1140, 1140, 1140, 1140, 896, 999, 999, - 999, 999, 999, 999, 999, 1044, 1469, 1395, 1469, 1000, - 1469, 1469, 1469, 1469, 1394, 1000, 1000, 1000, 1000, 1000, - 1000, 1141, 1142, 1142, 1142, 1142, 1142, 1142, 1469, 1469, - 1469, 1469, 1143, 1469, 1469, 1469, 1469, 1469, 1143, 1143, - 1143, 1143, 1143, 1143, 1145, 1146, 1146, 1146, 1146, 1146, - - 1146, 1146, 1147, 1469, 1469, 1469, 1148, 1469, 1149, 1469, - 1469, 1469, 1148, 1148, 1148, 1148, 1148, 1148, 1341, 1342, - 1342, 1342, 1342, 1342, 1342, 1347, 1348, 1348, 1348, 1348, - 1348, 1348, 1398, 1469, 1149, 1051, 1153, 1153, 1153, 1153, - 1153, 1153, 1153, 1053, 1469, 1469, 1399, 1154, 1469, 1469, - 1469, 1469, 1469, 1154, 1154, 1154, 1154, 1154, 1154, 1060, - 1163, 1163, 1163, 1163, 1163, 1163, 1163, 1062, 1469, 1469, - 1469, 1164, 1399, 1469, 1469, 1469, 1469, 1164, 1164, 1164, - 1164, 1164, 1164, 1071, 1176, 1176, 1176, 1176, 1176, 1176, - 1176, 1073, 1469, 1469, 1469, 1177, 1469, 1469, 1469, 1469, - - 1469, 1177, 1177, 1177, 1177, 1177, 1177, 1083, 1190, 1190, - 1190, 1190, 1190, 1190, 1190, 1085, 1469, 1469, 1469, 1191, - 1469, 1469, 1469, 1469, 1469, 1191, 1191, 1191, 1191, 1191, - 1191, 1095, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1097, - 1469, 1469, 1469, 1205, 1469, 1469, 1469, 1469, 1469, 1205, - 1205, 1205, 1205, 1205, 1205, 1012, 1213, 1213, 1213, 1213, - 1213, 1213, 1213, 1014, 1469, 1469, 1469, 1214, 1469, 1469, - 1469, 1469, 1469, 1214, 1214, 1214, 1214, 1214, 1214, 1215, - 1216, 1216, 1216, 1216, 1216, 1216, 1216, 1217, 1469, 1469, - 1469, 1218, 1469, 1219, 1469, 1469, 1469, 1218, 1218, 1218, - - 1218, 1218, 1218, 1350, 1351, 1351, 1351, 1351, 1351, 1351, - 1356, 1357, 1357, 1357, 1357, 1357, 1357, 1469, 1469, 1219, - 55, 607, 522, 522, 522, 522, 522, 522, 522, 1359, - 1360, 1360, 1360, 1360, 1360, 1360, 1469, 1398, 55, 1228, - 1229, 1229, 1229, 1229, 1229, 1229, 1230, 1469, 1469, 1469, - 1231, 1469, 1469, 1469, 1469, 1469, 1231, 1231, 1231, 1231, - 1231, 1231, 1051, 1232, 1232, 1232, 1232, 1232, 1232, 1232, - 1233, 1469, 1469, 1469, 1234, 1469, 1055, 1469, 1469, 1469, - 1234, 1234, 1234, 1234, 1234, 1234, 1293, 1293, 1293, 1293, - 1293, 1293, 1293, 1384, 1385, 1385, 1385, 1385, 1385, 1385, - - 1469, 1469, 1055, 1060, 1235, 1235, 1235, 1235, 1235, 1235, - 1235, 1236, 1469, 1469, 1469, 1237, 1469, 1064, 1469, 1469, - 1469, 1237, 1237, 1237, 1237, 1237, 1237, 1330, 1330, 1330, - 1330, 1330, 1330, 1330, 1339, 1339, 1339, 1339, 1339, 1339, - 1339, 1402, 1469, 1064, 1071, 1238, 1238, 1238, 1238, 1238, - 1238, 1238, 1239, 1469, 1469, 1403, 1240, 1469, 1075, 1469, - 1469, 1402, 1240, 1240, 1240, 1240, 1240, 1240, 1348, 1348, - 1348, 1348, 1348, 1348, 1348, 1469, 1469, 1469, 1469, 1469, - 1406, 1403, 1406, 1469, 1075, 1083, 1241, 1241, 1241, 1241, - 1241, 1241, 1241, 1242, 1407, 1469, 1469, 1243, 1469, 1087, - - 1469, 1469, 1422, 1243, 1243, 1243, 1243, 1243, 1243, 1357, - 1357, 1357, 1357, 1357, 1357, 1357, 1423, 1469, 1469, 1469, - 1407, 1469, 1469, 1469, 1469, 1087, 1095, 1244, 1244, 1244, - 1244, 1244, 1244, 1244, 1245, 1469, 1469, 1469, 1246, 1469, - 1099, 1469, 1423, 1469, 1246, 1246, 1246, 1246, 1246, 1246, - 1410, 1411, 1411, 1411, 1411, 1411, 1411, 1413, 1414, 1414, - 1414, 1414, 1414, 1414, 1422, 1469, 1099, 1145, 1249, 1249, - 1249, 1249, 1249, 1249, 1249, 1147, 1469, 1469, 1469, 1250, - 1469, 1469, 1469, 1469, 1469, 1250, 1250, 1250, 1250, 1250, - 1250, 1051, 1257, 1257, 1257, 1257, 1257, 1257, 1257, 1233, - - 1469, 1469, 1469, 1258, 1469, 1469, 1469, 1469, 1469, 1258, - 1258, 1258, 1258, 1258, 1258, 1060, 1264, 1264, 1264, 1264, - 1264, 1264, 1264, 1236, 1469, 1469, 1469, 1265, 1469, 1469, - 1469, 1469, 1469, 1265, 1265, 1265, 1265, 1265, 1265, 1071, - 1272, 1272, 1272, 1272, 1272, 1272, 1272, 1239, 1469, 1469, - 1469, 1273, 1469, 1469, 1469, 1469, 1469, 1273, 1273, 1273, - 1273, 1273, 1273, 1083, 1280, 1280, 1280, 1280, 1280, 1280, - 1280, 1242, 1469, 1469, 1469, 1281, 1469, 1469, 1469, 1469, - 1469, 1281, 1281, 1281, 1281, 1281, 1281, 1095, 1288, 1288, - 1288, 1288, 1288, 1288, 1288, 1245, 1469, 1469, 1469, 1289, - - 1469, 1469, 1469, 1469, 1469, 1289, 1289, 1289, 1289, 1289, - 1289, 1215, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1217, - 1469, 1469, 1469, 1302, 1469, 1469, 1469, 1469, 1469, 1302, - 1302, 1302, 1302, 1302, 1302, 1145, 1309, 1309, 1309, 1309, - 1309, 1309, 1309, 1310, 1469, 1469, 1469, 1311, 1469, 1149, - 1469, 1469, 1469, 1311, 1311, 1311, 1311, 1311, 1311, 1424, - 1425, 1425, 1425, 1425, 1425, 1425, 1428, 1429, 1429, 1429, - 1429, 1429, 1429, 1444, 1469, 1149, 1051, 1153, 1153, 1153, - 1153, 1153, 1153, 1153, 1233, 1469, 1469, 1445, 1154, 1469, - 1469, 1469, 1469, 1469, 1154, 1154, 1154, 1154, 1154, 1154, - - 1060, 1163, 1163, 1163, 1163, 1163, 1163, 1163, 1236, 1469, - 1469, 1469, 1164, 1445, 1469, 1469, 1469, 1469, 1164, 1164, - 1164, 1164, 1164, 1164, 1071, 1176, 1176, 1176, 1176, 1176, - 1176, 1176, 1239, 1469, 1469, 1469, 1177, 1469, 1469, 1469, - 1469, 1469, 1177, 1177, 1177, 1177, 1177, 1177, 1083, 1190, - 1190, 1190, 1190, 1190, 1190, 1190, 1242, 1469, 1469, 1469, - 1191, 1469, 1469, 1469, 1469, 1469, 1191, 1191, 1191, 1191, - 1191, 1191, 1095, 1204, 1204, 1204, 1204, 1204, 1204, 1204, - 1245, 1469, 1469, 1469, 1205, 1469, 1469, 1469, 1469, 1469, - 1205, 1205, 1205, 1205, 1205, 1205, 1145, 1320, 1320, 1320, - - 1320, 1320, 1320, 1320, 1310, 1469, 1469, 1469, 1321, 1469, - 1469, 1469, 1469, 1469, 1321, 1321, 1321, 1321, 1321, 1321, - 1215, 1369, 1369, 1369, 1369, 1369, 1369, 1369, 1217, 1469, - 1469, 1469, 1370, 1469, 1469, 1469, 1469, 1469, 1370, 1370, - 1370, 1370, 1370, 1370, 1145, 1249, 1249, 1249, 1249, 1249, - 1249, 1249, 1310, 1469, 1469, 1469, 1250, 1469, 1469, 1469, - 1469, 1469, 1250, 1250, 1250, 1250, 1250, 1250, 1432, 1433, - 1433, 1433, 1433, 1433, 1433, 1436, 1437, 1437, 1437, 1437, - 1437, 1437, 1440, 1441, 1441, 1441, 1441, 1441, 1441, 1411, - 1411, 1411, 1411, 1411, 1411, 1411, 1444, 1448, 1449, 1449, - - 1449, 1449, 1449, 1449, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1429, 1429, - 1429, 1429, 1429, 1429, 1429, 1433, 1433, 1433, 1433, 1433, - 1433, 1433, 1469, 1469, 1469, 1469, 1469, 1437, 1437, 1437, - 1437, 1437, 1437, 1437, 1441, 1441, 1441, 1441, 1441, 1441, - 1441, 1457, 1458, 1458, 1458, 1458, 1458, 1458, 1449, 1449, - 1449, 1449, 1449, 1449, 1449, 1458, 1458, 1458, 1458, 1458, - 1458, 1458, 46, 1469, 1469, 1469, 1469, 46, 46, 46, - 64, 1469, 64, 64, 64, 64, 64, 64, 64, 152, - 1469, 152, 159, 159, 159, 271, 271, 271, 280, 280, - - 280, 359, 359, 359, 362, 362, 362, 363, 363, 363, - 370, 370, 370, 368, 368, 368, 374, 374, 374, 378, - 1469, 378, 443, 443, 443, 448, 448, 448, 449, 449, - 449, 458, 458, 458, 462, 1469, 462, 463, 463, 463, - 372, 372, 1469, 1469, 372, 467, 467, 467, 471, 471, - 471, 362, 362, 362, 529, 529, 529, 533, 533, 533, - 534, 534, 534, 535, 535, 535, 370, 370, 370, 540, - 540, 540, 456, 456, 1469, 1469, 456, 545, 545, 545, - 549, 549, 549, 553, 1469, 553, 554, 554, 554, 558, - 558, 558, 562, 1469, 562, 612, 612, 612, 458, 458, - - 458, 620, 620, 620, 621, 621, 621, 629, 629, 629, - 633, 1469, 633, 636, 1469, 636, 637, 637, 637, 641, - 641, 641, 645, 1469, 645, 552, 552, 1469, 1469, 552, - 556, 556, 1469, 1469, 556, 651, 651, 651, 655, 655, - 655, 562, 562, 1469, 562, 534, 534, 534, 689, 689, - 689, 693, 693, 693, 696, 696, 696, 697, 697, 697, - 698, 698, 698, 703, 703, 703, 627, 627, 1469, 1469, - 627, 708, 708, 708, 712, 712, 712, 633, 633, 1469, - 633, 635, 635, 1469, 1469, 635, 636, 636, 1469, 636, - 637, 637, 639, 639, 1469, 1469, 639, 719, 719, 719, - - 723, 723, 723, 645, 645, 1469, 645, 727, 1469, 727, - 730, 1469, 730, 731, 731, 731, 735, 735, 735, 739, - 1469, 739, 766, 766, 766, 629, 629, 629, 641, 641, - 641, 777, 777, 777, 778, 778, 778, 786, 786, 786, - 790, 1469, 790, 793, 1469, 793, 794, 794, 794, 798, - 798, 798, 802, 1469, 802, 805, 1469, 805, 808, 1469, - 808, 809, 809, 809, 813, 813, 813, 817, 1469, 817, - 726, 1469, 1469, 726, 727, 727, 1469, 727, 729, 729, - 1469, 1469, 729, 730, 730, 1469, 730, 731, 731, 733, - 733, 1469, 1469, 733, 824, 824, 824, 828, 828, 828, - - 739, 739, 1469, 739, 53, 53, 53, 1469, 53, 53, - 697, 697, 697, 850, 850, 850, 854, 854, 854, 857, - 857, 857, 860, 860, 860, 861, 861, 861, 862, 862, - 862, 867, 867, 867, 784, 784, 1469, 1469, 784, 872, - 872, 872, 876, 876, 876, 790, 790, 1469, 790, 792, - 792, 1469, 1469, 792, 793, 793, 1469, 793, 794, 794, - 796, 796, 1469, 1469, 796, 883, 883, 883, 887, 887, - 887, 802, 802, 1469, 802, 804, 1469, 1469, 804, 805, - 805, 1469, 805, 807, 807, 1469, 1469, 807, 808, 808, - 1469, 808, 809, 809, 811, 811, 1469, 1469, 811, 895, - - 895, 895, 899, 899, 899, 817, 817, 1469, 817, 901, - 1469, 901, 904, 1469, 904, 907, 1469, 907, 908, 908, - 908, 912, 912, 912, 916, 1469, 916, 53, 53, 53, - 1469, 53, 53, 934, 934, 934, 786, 786, 786, 798, - 798, 798, 813, 813, 813, 948, 948, 948, 949, 949, - 949, 957, 957, 957, 961, 1469, 961, 964, 1469, 964, - 965, 965, 965, 969, 969, 969, 973, 1469, 973, 976, - 1469, 976, 979, 1469, 979, 980, 980, 980, 984, 984, - 984, 988, 1469, 988, 989, 1469, 989, 992, 1469, 992, - 995, 1469, 995, 996, 996, 996, 1000, 1000, 1000, 1004, - - 1469, 1004, 901, 1469, 901, 903, 1469, 1469, 903, 904, - 904, 1469, 904, 906, 906, 1469, 1469, 906, 907, 907, - 1469, 907, 908, 908, 910, 910, 1469, 1469, 910, 1011, - 1011, 1011, 1015, 1015, 1015, 916, 916, 1469, 916, 53, - 53, 53, 1469, 53, 53, 861, 861, 861, 1032, 1032, - 1032, 1036, 1036, 1036, 1039, 1039, 1039, 1042, 1042, 1042, - 1045, 1045, 1045, 1046, 1046, 1046, 1054, 1054, 1054, 955, - 955, 1469, 1469, 955, 1059, 1059, 1059, 1063, 1063, 1063, - 961, 961, 1469, 961, 963, 963, 1469, 1469, 963, 964, - 964, 1469, 964, 965, 965, 967, 967, 1469, 1469, 967, - - 1070, 1070, 1070, 1074, 1074, 1074, 973, 973, 1469, 973, - 975, 1469, 1469, 975, 976, 976, 1469, 976, 978, 978, - 1469, 1469, 978, 979, 979, 1469, 979, 980, 980, 982, - 982, 1469, 1469, 982, 1082, 1082, 1082, 1086, 1086, 1086, - 988, 988, 1469, 988, 989, 1469, 989, 991, 1469, 1469, - 991, 992, 992, 1469, 992, 994, 994, 1469, 1469, 994, - 995, 995, 1469, 995, 996, 996, 998, 998, 1469, 1469, - 998, 1094, 1094, 1094, 1098, 1098, 1098, 1004, 1004, 1469, - 1004, 1100, 1469, 1100, 1103, 1469, 1103, 1106, 1469, 1106, - 1107, 1107, 1107, 1111, 1111, 1111, 1115, 1469, 1115, 53, - - 53, 53, 1469, 53, 53, 1127, 1127, 1127, 957, 957, - 957, 969, 969, 969, 984, 984, 984, 1000, 1000, 1000, - 1144, 1144, 1144, 1150, 1150, 1150, 1148, 1148, 1148, 1155, - 1155, 1155, 1154, 1154, 1154, 1156, 1469, 1156, 1159, 1469, - 1159, 1160, 1160, 1160, 1165, 1165, 1165, 1164, 1164, 1164, - 1166, 1469, 1166, 1169, 1469, 1169, 1172, 1469, 1172, 1173, - 1173, 1173, 1178, 1178, 1178, 1177, 1177, 1177, 1179, 1469, - 1179, 1180, 1469, 1180, 1183, 1469, 1183, 1186, 1469, 1186, - 1187, 1187, 1187, 1192, 1192, 1192, 1191, 1191, 1191, 1193, - 1469, 1193, 1194, 1469, 1194, 1197, 1469, 1197, 1200, 1469, - - 1200, 1201, 1201, 1201, 1206, 1206, 1206, 1205, 1205, 1205, - 1207, 1469, 1207, 1100, 1469, 1100, 1102, 1469, 1469, 1102, - 1103, 1103, 1469, 1103, 1105, 1105, 1469, 1469, 1105, 1106, - 1106, 1469, 1106, 1107, 1107, 1109, 1109, 1469, 1469, 1109, - 1214, 1214, 1214, 1218, 1218, 1218, 1115, 1115, 1469, 1115, - 53, 53, 53, 1469, 53, 53, 1046, 1046, 1046, 1234, - 1234, 1234, 1237, 1237, 1237, 1240, 1240, 1240, 1243, 1243, - 1243, 1246, 1246, 1246, 1251, 1251, 1251, 1250, 1250, 1250, - 1253, 1469, 1253, 1254, 1254, 1254, 1152, 1152, 1469, 1469, - 1152, 1258, 1258, 1258, 1259, 1259, 1259, 1156, 1156, 1469, - - 1156, 1158, 1158, 1469, 1469, 1158, 1159, 1159, 1469, 1159, - 1160, 1160, 1162, 1162, 1469, 1469, 1162, 1265, 1265, 1265, - 1266, 1266, 1266, 1166, 1166, 1469, 1166, 1168, 1469, 1469, - 1168, 1169, 1169, 1469, 1169, 1171, 1171, 1469, 1469, 1171, - 1172, 1172, 1469, 1172, 1173, 1173, 1175, 1175, 1469, 1469, - 1175, 1273, 1273, 1273, 1274, 1274, 1274, 1179, 1179, 1469, - 1179, 1180, 1469, 1180, 1182, 1469, 1469, 1182, 1183, 1183, - 1469, 1183, 1185, 1185, 1469, 1469, 1185, 1186, 1186, 1469, - 1186, 1187, 1187, 1189, 1189, 1469, 1469, 1189, 1281, 1281, - 1281, 1282, 1282, 1282, 1193, 1193, 1469, 1193, 1194, 1469, - - 1194, 1196, 1469, 1469, 1196, 1197, 1197, 1469, 1197, 1199, - 1199, 1469, 1469, 1199, 1200, 1200, 1469, 1200, 1201, 1201, - 1203, 1203, 1469, 1469, 1203, 1289, 1289, 1289, 1290, 1290, - 1290, 1207, 1207, 1469, 1207, 1291, 1469, 1291, 1294, 1469, - 1294, 1297, 1469, 1297, 1298, 1298, 1298, 1303, 1469, 1303, - 1302, 1302, 1302, 1304, 1469, 1304, 53, 53, 53, 1469, - 53, 53, 1312, 1469, 1312, 1311, 1311, 1311, 1313, 1469, - 1313, 1154, 1154, 1154, 1314, 1469, 1314, 1164, 1164, 1164, - 1315, 1469, 1315, 1177, 1177, 1177, 1316, 1469, 1316, 1191, - 1191, 1191, 1317, 1469, 1317, 1205, 1205, 1205, 1248, 1248, - - 1469, 1469, 1248, 1321, 1321, 1321, 1322, 1322, 1322, 370, - 370, 370, 1253, 1253, 1469, 1253, 1323, 1323, 1323, 1326, - 1469, 1326, 1327, 1327, 1327, 1328, 1328, 1328, 1331, 1469, - 1331, 1334, 1469, 1334, 1335, 1335, 1335, 1336, 1336, 1336, - 1337, 1469, 1337, 1340, 1469, 1340, 1343, 1469, 1343, 1344, - 1344, 1344, 1345, 1345, 1345, 1346, 1469, 1346, 1349, 1469, - 1349, 1352, 1469, 1352, 1353, 1353, 1353, 1354, 1354, 1354, - 1355, 1469, 1355, 1358, 1469, 1358, 1361, 1469, 1361, 1362, - 1362, 1362, 1363, 1363, 1363, 1291, 1469, 1291, 1293, 1469, - 1469, 1293, 1294, 1294, 1469, 1294, 1296, 1296, 1469, 1469, - - 1296, 1297, 1297, 1469, 1297, 1298, 1298, 1300, 1300, 1469, - 1469, 1300, 1370, 1370, 1370, 1371, 1469, 1371, 1304, 1304, - 1469, 1304, 53, 53, 53, 1469, 53, 53, 1376, 1376, - 1376, 1250, 1250, 1250, 1378, 1469, 1378, 1379, 1469, 1379, - 1380, 1469, 1380, 1381, 1469, 1381, 1382, 1469, 1382, 1383, - 1469, 1383, 1386, 1469, 1386, 1387, 1387, 1387, 1388, 1388, - 1388, 1389, 1469, 1389, 1325, 1325, 1469, 1469, 1325, 1326, - 1326, 1469, 1326, 1327, 1327, 1392, 1469, 1392, 1330, 1469, - 1469, 1330, 1331, 1331, 1469, 1331, 1333, 1333, 1469, 1469, - 1333, 1334, 1334, 1469, 1334, 1335, 1335, 1396, 1469, 1396, - - 1337, 1469, 1337, 1339, 1469, 1469, 1339, 1340, 1340, 1469, - 1340, 1342, 1342, 1469, 1469, 1342, 1343, 1343, 1469, 1343, - 1344, 1344, 1400, 1469, 1400, 1346, 1469, 1346, 1348, 1469, - 1469, 1348, 1349, 1349, 1469, 1349, 1351, 1351, 1469, 1469, - 1351, 1352, 1352, 1469, 1352, 1353, 1353, 1404, 1469, 1404, - 1355, 1469, 1355, 1357, 1469, 1469, 1357, 1358, 1358, 1469, - 1358, 1360, 1360, 1469, 1469, 1360, 1361, 1361, 1469, 1361, - 1362, 1362, 1408, 1469, 1408, 1409, 1469, 1409, 1412, 1469, - 1412, 1415, 1469, 1415, 1416, 1416, 1416, 1417, 1469, 1417, - 53, 53, 53, 1469, 53, 53, 1421, 1469, 1421, 1323, - - 1469, 1323, 1328, 1469, 1328, 1336, 1469, 1336, 1345, 1469, - 1345, 1354, 1469, 1354, 1363, 1469, 1363, 1385, 1385, 1469, - 1469, 1385, 1386, 1386, 1469, 1386, 1387, 1387, 1377, 1469, - 1377, 1426, 1469, 1426, 1427, 1469, 1427, 1430, 1469, 1430, - 1431, 1469, 1431, 1434, 1469, 1434, 1435, 1469, 1435, 1438, - 1469, 1438, 1439, 1469, 1439, 1442, 1469, 1442, 1411, 1469, - 1469, 1411, 1414, 1414, 1469, 1469, 1414, 1446, 1469, 1446, - 1388, 1469, 1388, 1450, 1469, 1450, 1425, 1469, 1469, 1425, - 1429, 1469, 1469, 1429, 1433, 1469, 1469, 1433, 1437, 1469, - 1469, 1437, 1441, 1469, 1469, 1441, 1456, 1469, 1456, 1459, - - 1469, 1459, 1449, 1469, 1469, 1449, 1461, 1469, 1461, 1462, - 1469, 1462, 1463, 1469, 1463, 1464, 1469, 1464, 1465, 1469, - 1465, 1458, 1469, 1469, 1458, 1467, 1469, 1467, 1468, 1469, - 1468, 3, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469 + 469, 469, 469, 469, 469, 470, 55, 975, 55, 471, + 787, 472, 1376, 965, 788, 471, 471, 471, 471, 471, + 471, 582, 738, 583, 53, 789, 584, 585, 53, 738, + 586, 587, 791, 733, 792, 588, 739, 472, 482, 53, + 1161, 902, 483, 1728, 53, 484, 53, 740, 485, 740, + 486, 487, 488, 489, 521, 521, 521, 521, 521, 521, + 521, 741, 739, 1728, 1295, 521, 55, 55, 55, 1728, + + 1421, 521, 521, 521, 521, 521, 521, 522, 523, 523, + 523, 523, 523, 523, 55, 55, 55, 741, 524, 1728, + 967, 969, 1061, 880, 524, 524, 524, 524, 524, 524, + 432, 525, 525, 525, 525, 525, 525, 525, 526, 526, + 526, 526, 526, 526, 526, 1223, 1468, 1470, 1061, 526, + 1506, 55, 55, 55, 55, 526, 526, 526, 526, 526, + 526, 437, 528, 528, 528, 528, 528, 528, 528, 55, + 55, 55, 55, 529, 1510, 265, 970, 976, 881, 529, + 529, 529, 529, 529, 529, 646, 647, 647, 647, 647, + 647, 647, 650, 651, 651, 651, 651, 651, 651, 55, + + 855, 265, 529, 529, 529, 529, 529, 529, 529, 1728, + 744, 1728, 1728, 529, 856, 55, 1514, 55, 905, 529, + 529, 529, 529, 529, 529, 530, 530, 530, 530, 530, + 530, 530, 1518, 55, 867, 1728, 531, 1728, 1728, 1728, + 856, 979, 531, 531, 531, 531, 531, 531, 53, 531, + 531, 531, 531, 531, 531, 531, 365, 534, 534, 534, + 534, 534, 534, 534, 535, 1728, 1522, 1551, 536, 1728, + 369, 1573, 1155, 1149, 536, 536, 536, 536, 536, 536, + 749, 750, 750, 750, 750, 750, 750, 752, 753, 753, + 753, 753, 753, 753, 55, 1728, 369, 540, 541, 541, + + 541, 541, 541, 541, 541, 542, 861, 453, 1332, 543, + 930, 544, 55, 1085, 863, 543, 543, 543, 543, 543, + 543, 756, 757, 757, 757, 757, 757, 757, 819, 820, + 820, 820, 820, 820, 820, 855, 55, 544, 365, 547, + 547, 547, 547, 547, 547, 547, 535, 1145, 1143, 1728, + 548, 1728, 1137, 1728, 55, 55, 548, 548, 548, 548, + 548, 548, 549, 550, 550, 550, 550, 550, 550, 550, + 551, 868, 453, 55, 552, 1728, 553, 1728, 982, 1728, + 552, 552, 552, 552, 552, 552, 827, 828, 828, 828, + 828, 828, 828, 831, 832, 832, 832, 832, 832, 832, + + 857, 55, 553, 468, 560, 560, 560, 560, 560, 560, + 560, 470, 1329, 1728, 858, 561, 919, 55, 1133, 55, + 857, 561, 561, 561, 561, 561, 561, 562, 563, 563, + 563, 563, 563, 563, 1728, 55, 869, 1131, 564, 1728, + 858, 55, 1087, 55, 564, 564, 564, 564, 564, 564, + 613, 915, 614, 614, 614, 614, 614, 614, 614, 55, + 1728, 55, 1126, 614, 972, 916, 55, 55, 885, 614, + 614, 614, 614, 614, 614, 615, 616, 616, 616, 616, + 616, 616, 616, 453, 55, 55, 1326, 617, 1078, 618, + 910, 916, 1084, 617, 617, 617, 617, 617, 617, 839, + + 840, 840, 840, 840, 840, 840, 842, 843, 843, 843, + 843, 843, 843, 55, 55, 618, 615, 617, 617, 617, + 617, 617, 617, 617, 1142, 910, 1122, 1728, 617, 1154, + 1728, 55, 55, 1101, 617, 617, 617, 617, 617, 617, + 437, 620, 620, 620, 620, 620, 620, 620, 871, 919, + 1142, 877, 621, 1728, 1728, 1154, 1728, 1104, 621, 621, + 621, 621, 621, 621, 53, 621, 621, 621, 621, 621, + 621, 621, 622, 622, 622, 622, 622, 622, 622, 930, + 1728, 1117, 453, 623, 1286, 1728, 1728, 1107, 1323, 623, + 623, 623, 623, 623, 623, 53, 623, 623, 623, 623, + + 623, 623, 623, 626, 627, 627, 627, 627, 627, 627, + 1286, 1728, 1728, 1728, 628, 453, 1225, 55, 55, 55, + 628, 628, 628, 628, 628, 628, 365, 457, 457, 457, + 457, 457, 457, 457, 535, 55, 55, 55, 458, 1728, + 1082, 1086, 1089, 915, 458, 458, 458, 458, 458, 458, + 629, 630, 630, 630, 630, 630, 630, 1728, 942, 1342, + 1728, 631, 1206, 55, 55, 1335, 1110, 631, 631, 631, + 631, 631, 631, 635, 636, 636, 636, 636, 636, 636, + 453, 55, 55, 1728, 637, 1090, 1728, 55, 55, 1088, + 637, 637, 637, 637, 637, 637, 540, 640, 640, 640, + + 640, 640, 640, 640, 542, 55, 55, 1091, 641, 1332, + 55, 1093, 55, 1329, 641, 641, 641, 641, 641, 641, + 642, 643, 643, 643, 643, 643, 643, 453, 55, 1326, + 55, 644, 1323, 1206, 55, 55, 55, 644, 644, 644, + 644, 644, 644, 549, 652, 652, 652, 652, 652, 652, + 652, 551, 55, 55, 55, 653, 1068, 55, 55, 55, + 1295, 653, 653, 653, 653, 653, 653, 654, 655, 655, + 655, 655, 655, 655, 453, 55, 55, 55, 656, 1187, + 1190, 55, 1191, 55, 656, 656, 656, 656, 656, 656, + 468, 662, 662, 662, 662, 662, 662, 662, 470, 55, + + 1070, 55, 663, 1728, 1728, 55, 1194, 861, 663, 663, + 663, 663, 663, 663, 664, 665, 665, 665, 665, 665, + 665, 665, 666, 55, 1064, 1195, 667, 1062, 668, 1728, + 1728, 1163, 667, 667, 667, 667, 667, 667, 846, 847, + 847, 847, 847, 847, 847, 750, 750, 750, 750, 750, + 750, 750, 55, 55, 668, 704, 704, 704, 704, 704, + 704, 704, 1357, 453, 1365, 1373, 704, 1728, 55, 1110, + 55, 55, 704, 704, 704, 704, 704, 704, 705, 705, + 705, 705, 705, 705, 705, 878, 55, 968, 1357, 705, + 1365, 1373, 1199, 1728, 1151, 705, 705, 705, 705, 705, + + 705, 615, 706, 706, 706, 706, 706, 706, 706, 707, + 707, 707, 707, 707, 707, 707, 1068, 453, 1107, 1467, + 707, 1509, 55, 55, 1070, 55, 707, 707, 707, 707, + 707, 707, 437, 709, 709, 709, 709, 709, 709, 709, + 55, 55, 1197, 55, 710, 1467, 1200, 1509, 1513, 978, + 710, 710, 710, 710, 710, 710, 53, 710, 710, 710, + 710, 710, 710, 710, 711, 711, 711, 711, 711, 711, + 711, 1117, 1126, 1139, 1513, 712, 453, 1104, 1128, 1323, + 1326, 712, 712, 712, 712, 712, 712, 53, 712, 712, + 712, 712, 712, 712, 712, 540, 715, 715, 715, 715, + + 715, 715, 715, 716, 55, 55, 924, 717, 55, 544, + 55, 55, 55, 717, 717, 717, 717, 717, 717, 1137, + 925, 453, 55, 55, 1101, 1119, 55, 1329, 55, 55, + 55, 55, 980, 55, 870, 544, 549, 718, 718, 718, + 718, 718, 718, 718, 719, 882, 925, 886, 720, 55, + 553, 55, 1149, 1298, 720, 720, 720, 720, 720, 720, + 1332, 875, 453, 1225, 888, 893, 893, 893, 893, 893, + 893, 893, 55, 924, 926, 1517, 553, 724, 725, 725, + 725, 725, 725, 725, 725, 726, 55, 1728, 927, 727, + 55, 728, 926, 936, 936, 727, 727, 727, 727, 727, + + 727, 1517, 341, 1161, 55, 1521, 1728, 937, 1728, 55, + 453, 1335, 1206, 1728, 927, 938, 1110, 728, 540, 731, + 731, 731, 731, 731, 731, 731, 716, 55, 1299, 939, + 732, 1521, 1728, 937, 1728, 55, 732, 732, 732, 732, + 732, 732, 733, 734, 734, 734, 734, 734, 734, 734, + 735, 1300, 1107, 55, 736, 939, 737, 1104, 1304, 938, + 736, 736, 736, 736, 736, 736, 840, 840, 840, 840, + 840, 840, 840, 1728, 948, 949, 949, 949, 949, 949, + 949, 1101, 737, 549, 742, 742, 742, 742, 742, 742, + 742, 719, 1223, 1206, 1572, 743, 1592, 1593, 55, 1728, + + 1413, 743, 743, 743, 743, 743, 743, 744, 745, 745, + 745, 745, 745, 745, 745, 746, 55, 1070, 1301, 747, + 1572, 748, 1592, 1593, 863, 747, 747, 747, 747, 747, + 747, 951, 952, 952, 952, 952, 952, 952, 955, 956, + 956, 956, 956, 956, 956, 55, 55, 748, 664, 758, + 758, 758, 758, 758, 758, 758, 666, 1293, 942, 1163, + 759, 1110, 1594, 55, 55, 1295, 759, 759, 759, 759, + 759, 759, 760, 761, 761, 761, 761, 761, 761, 966, + 974, 744, 938, 762, 55, 55, 55, 55, 1594, 762, + 762, 762, 762, 762, 762, 53, 53, 53, 53, 53, + + 53, 53, 55, 55, 55, 55, 53, 55, 55, 1062, + 55, 55, 53, 53, 53, 53, 53, 53, 971, 973, + 1595, 983, 981, 1063, 1293, 55, 55, 1062, 55, 55, + 764, 794, 1728, 795, 795, 795, 795, 795, 795, 795, + 977, 1728, 1077, 1073, 795, 1186, 1595, 55, 936, 1063, + 795, 795, 795, 795, 795, 795, 1010, 1011, 1011, 1011, + 1011, 1011, 1011, 1064, 1064, 55, 55, 1728, 53, 796, + 797, 797, 797, 797, 797, 797, 797, 1065, 1728, 930, + 1074, 797, 55, 55, 55, 1079, 1151, 797, 797, 797, + 797, 797, 797, 1018, 1019, 1019, 1019, 1019, 1019, 1019, + + 55, 55, 1080, 1065, 1728, 53, 706, 706, 706, 706, + 706, 706, 706, 1022, 1023, 1023, 1023, 1023, 1023, 1023, + 1030, 1031, 1031, 1031, 1031, 1031, 1031, 1033, 1034, 1034, + 1034, 1034, 1034, 1034, 1037, 1038, 1038, 1038, 1038, 1038, + 1038, 53, 707, 707, 707, 707, 707, 707, 707, 1107, + 733, 926, 924, 707, 919, 1596, 1613, 1619, 1139, 707, + 707, 707, 707, 707, 707, 798, 353, 353, 353, 353, + 353, 353, 353, 267, 800, 800, 800, 800, 800, 800, + 800, 1596, 1613, 1619, 1104, 801, 724, 915, 910, 1128, + 1101, 801, 801, 801, 801, 801, 801, 53, 801, 801, + + 801, 801, 801, 801, 801, 804, 805, 805, 805, 805, + 805, 805, 1119, 1006, 991, 1110, 806, 1107, 1104, 55, + 55, 55, 806, 806, 806, 806, 806, 806, 540, 640, + 640, 640, 640, 640, 640, 640, 716, 55, 55, 55, + 641, 1315, 1101, 1316, 55, 1122, 641, 641, 641, 641, + 641, 641, 807, 808, 808, 808, 808, 808, 808, 1123, + 1302, 1099, 55, 809, 991, 861, 55, 1070, 55, 809, + 809, 809, 809, 809, 809, 549, 652, 652, 652, 652, + 652, 652, 652, 719, 55, 1123, 55, 653, 863, 55, + 55, 55, 1122, 653, 653, 653, 653, 653, 653, 810, + + 811, 811, 811, 811, 811, 811, 1728, 55, 55, 55, + 812, 664, 55, 1395, 1404, 1306, 812, 812, 812, 812, + 812, 812, 816, 817, 817, 817, 817, 817, 817, 453, + 55, 857, 1728, 818, 855, 944, 905, 55, 55, 818, + 818, 818, 818, 818, 818, 724, 821, 821, 821, 821, + 821, 821, 821, 726, 1401, 55, 55, 822, 932, 902, + 55, 55, 921, 822, 822, 822, 822, 822, 822, 823, + 824, 824, 824, 824, 824, 824, 453, 1409, 55, 55, + 825, 1476, 1477, 55, 55, 55, 825, 825, 825, 825, + 825, 825, 733, 833, 833, 833, 833, 833, 833, 833, + + 735, 55, 55, 55, 834, 899, 1479, 912, 55, 1006, + 834, 834, 834, 834, 834, 834, 835, 836, 836, 836, + 836, 836, 836, 453, 991, 1478, 55, 837, 905, 902, + 1481, 55, 55, 837, 837, 837, 837, 837, 837, 744, + 848, 848, 848, 848, 848, 848, 848, 746, 899, 55, + 55, 849, 991, 1482, 55, 55, 1483, 849, 849, 849, + 849, 849, 849, 850, 851, 851, 851, 851, 851, 851, + 453, 863, 55, 55, 852, 666, 1486, 55, 1489, 55, + 852, 852, 852, 852, 852, 852, 664, 859, 859, 859, + 859, 859, 859, 859, 666, 55, 744, 55, 860, 944, + + 905, 55, 549, 1536, 860, 860, 860, 860, 860, 860, + 861, 862, 862, 862, 862, 862, 862, 862, 863, 55, + 1546, 740, 864, 738, 865, 733, 932, 902, 864, 864, + 864, 864, 864, 864, 1046, 1047, 1047, 1047, 1047, 1047, + 1047, 1049, 1050, 1050, 1050, 1050, 1050, 1050, 1131, 55, + 865, 55, 1131, 889, 889, 889, 889, 889, 889, 889, + 540, 729, 1132, 724, 889, 921, 1728, 55, 55, 55, + 889, 889, 889, 889, 889, 889, 890, 890, 890, 890, + 890, 890, 890, 1081, 899, 912, 55, 890, 1132, 55, + 55, 55, 1728, 890, 890, 890, 890, 890, 890, 891, + + 892, 892, 892, 892, 892, 892, 1189, 55, 55, 55, + 893, 1549, 55, 55, 55, 55, 893, 893, 893, 893, + 893, 893, 267, 894, 894, 894, 894, 894, 894, 894, + 55, 55, 55, 55, 895, 909, 1577, 1584, 815, 1396, + 895, 895, 895, 895, 895, 895, 53, 895, 895, 895, + 895, 895, 895, 895, 724, 898, 898, 898, 898, 898, + 898, 898, 899, 803, 905, 902, 900, 899, 728, 897, + 803, 664, 900, 900, 900, 900, 900, 900, 1053, 1054, + 1054, 1054, 1054, 1054, 1054, 949, 949, 949, 949, 949, + 949, 949, 55, 55, 728, 733, 901, 901, 901, 901, + + 901, 901, 901, 902, 55, 55, 1133, 903, 863, 737, + 55, 55, 55, 903, 903, 903, 903, 903, 903, 666, + 1134, 468, 55, 55, 660, 1075, 1076, 55, 1083, 55, + 55, 55, 658, 1182, 1183, 737, 744, 904, 904, 904, + 904, 904, 904, 904, 905, 55, 1134, 55, 906, 55, + 748, 1537, 1133, 746, 906, 906, 906, 906, 906, 906, + 1092, 1143, 1094, 1185, 719, 735, 1728, 1031, 1031, 1031, + 1031, 1031, 1031, 1031, 1143, 1144, 748, 910, 911, 911, + 911, 911, 911, 911, 911, 912, 55, 55, 1728, 913, + 716, 914, 1728, 1145, 1145, 913, 913, 913, 913, 913, + + 913, 1144, 726, 815, 55, 55, 803, 1146, 1728, 719, + 55, 716, 803, 1155, 1728, 1155, 1184, 914, 724, 917, + 917, 917, 917, 917, 917, 917, 899, 1156, 55, 1728, + 918, 53, 53, 1146, 1728, 53, 918, 918, 918, 918, + 918, 918, 919, 920, 920, 920, 920, 920, 920, 920, + 921, 1581, 790, 1156, 922, 1728, 923, 55, 55, 1157, + 922, 922, 922, 922, 922, 922, 1047, 1047, 1047, 1047, + 1047, 1047, 1047, 1158, 53, 55, 55, 53, 1157, 55, + 55, 772, 923, 733, 928, 928, 928, 928, 928, 928, + 928, 902, 1728, 1310, 1192, 929, 666, 55, 55, 1158, + + 470, 929, 929, 929, 929, 929, 929, 930, 931, 931, + 931, 931, 931, 931, 931, 932, 1311, 1490, 1728, 933, + 549, 934, 746, 719, 365, 933, 933, 933, 933, 933, + 933, 1167, 1168, 1168, 1168, 1168, 1168, 1168, 1170, 1171, + 1171, 1171, 1171, 1171, 1171, 1287, 55, 934, 744, 940, + 940, 940, 940, 940, 940, 940, 905, 545, 540, 1288, + 941, 735, 716, 726, 55, 1201, 941, 941, 941, 941, + 941, 941, 942, 943, 943, 943, 943, 943, 943, 943, + 944, 1202, 1203, 723, 945, 1288, 946, 55, 55, 55, + 945, 945, 945, 945, 945, 945, 1174, 1175, 1175, 1175, + + 1175, 1175, 1175, 634, 625, 55, 55, 55, 55, 1287, + 1289, 719, 946, 861, 957, 957, 957, 957, 957, 957, + 957, 863, 1193, 1728, 1290, 958, 55, 1544, 55, 55, + 1289, 958, 958, 958, 958, 958, 958, 959, 960, 960, + 960, 960, 960, 960, 1728, 1314, 55, 55, 961, 1728, + 1290, 55, 716, 55, 961, 961, 961, 961, 961, 961, + 55, 1188, 984, 984, 984, 984, 984, 984, 984, 55, + 1728, 55, 1602, 984, 1603, 55, 55, 55, 55, 984, + 984, 984, 984, 984, 984, 985, 985, 985, 985, 985, + 985, 985, 714, 55, 55, 55, 985, 1604, 55, 55, + + 55, 55, 985, 985, 985, 985, 985, 985, 619, 986, + 986, 986, 986, 986, 986, 986, 55, 55, 55, 55, + 987, 1610, 435, 625, 615, 1491, 987, 987, 987, 987, + 987, 987, 1196, 1198, 1229, 1230, 1230, 1230, 1230, 1230, + 1230, 1235, 1236, 1236, 1236, 1236, 1236, 1236, 435, 987, + 987, 987, 987, 987, 987, 987, 613, 703, 702, 701, + 987, 700, 55, 55, 55, 55, 987, 987, 987, 987, + 987, 987, 267, 988, 988, 988, 988, 988, 988, 988, + 55, 55, 55, 55, 989, 1624, 699, 1626, 1627, 1535, + 989, 989, 989, 989, 989, 989, 53, 989, 989, 989, + + 989, 989, 989, 989, 992, 993, 993, 993, 993, 993, + 993, 698, 697, 696, 695, 994, 694, 693, 55, 55, + 55, 994, 994, 994, 994, 994, 994, 724, 821, 821, + 821, 821, 821, 821, 821, 899, 55, 55, 55, 822, + 692, 685, 1642, 55, 1345, 822, 822, 822, 822, 822, + 822, 995, 996, 996, 996, 996, 996, 996, 1346, 1634, + 684, 55, 997, 683, 682, 55, 1643, 55, 997, 997, + 997, 997, 997, 997, 733, 833, 833, 833, 833, 833, + 833, 833, 902, 55, 1346, 55, 834, 681, 1644, 55, + 55, 1345, 834, 834, 834, 834, 834, 834, 998, 999, + + 999, 999, 999, 999, 999, 1728, 1635, 55, 55, 1000, + 680, 679, 1646, 678, 55, 1000, 1000, 1000, 1000, 1000, + 1000, 744, 848, 848, 848, 848, 848, 848, 848, 905, + 677, 1728, 55, 849, 676, 675, 55, 55, 1350, 849, + 849, 849, 849, 849, 849, 1001, 1002, 1002, 1002, 1002, + 1002, 1002, 1351, 1636, 55, 55, 1003, 1655, 55, 1657, + 674, 673, 1003, 1003, 1003, 1003, 1003, 1003, 1007, 1008, + 1008, 1008, 1008, 1008, 1008, 453, 55, 672, 1351, 1009, + 671, 1659, 55, 55, 55, 1009, 1009, 1009, 1009, 1009, + 1009, 910, 1012, 1012, 1012, 1012, 1012, 1012, 1012, 912, + + 55, 55, 55, 1013, 670, 1661, 55, 669, 468, 1013, + 1013, 1013, 1013, 1013, 1013, 1014, 1015, 1015, 1015, 1015, + 1015, 1015, 453, 1669, 55, 1662, 1016, 666, 470, 55, + 464, 55, 1016, 1016, 1016, 1016, 1016, 1016, 919, 1024, + 1024, 1024, 1024, 1024, 1024, 1024, 921, 55, 1663, 55, + 1025, 551, 1672, 55, 55, 535, 1025, 1025, 1025, 1025, + 1025, 1025, 1026, 1027, 1027, 1027, 1027, 1027, 1027, 453, + 542, 55, 55, 1028, 634, 625, 1680, 1681, 55, 1028, + 1028, 1028, 1028, 1028, 1028, 930, 1039, 1039, 1039, 1039, + 1039, 1039, 1039, 932, 535, 625, 55, 1040, 619, 55, + + 55, 1686, 619, 1040, 1040, 1040, 1040, 1040, 1040, 1041, + 1042, 1042, 1042, 1042, 1042, 1042, 453, 55, 55, 612, + 1043, 1687, 611, 55, 55, 55, 1043, 1043, 1043, 1043, + 1043, 1043, 942, 1055, 1055, 1055, 1055, 1055, 1055, 1055, + 944, 55, 55, 55, 1056, 1693, 1690, 55, 55, 610, + 1056, 1056, 1056, 1056, 1056, 1056, 1057, 1058, 1058, 1058, + 1058, 1058, 1058, 453, 609, 55, 55, 1059, 608, 607, + 55, 1700, 55, 1059, 1059, 1059, 1059, 1059, 1059, 861, + 1066, 1066, 1066, 1066, 1066, 1066, 1066, 863, 55, 1694, + 55, 1067, 606, 605, 55, 1708, 55, 1067, 1067, 1067, + + 1067, 1067, 1067, 1068, 1069, 1069, 1069, 1069, 1069, 1069, + 1069, 1070, 55, 604, 55, 1071, 603, 1072, 602, 338, + 239, 1071, 1071, 1071, 1071, 1071, 1071, 1239, 1240, 1240, + 1240, 1240, 1240, 1240, 1245, 1246, 1246, 1246, 1246, 1246, + 1246, 55, 55, 1072, 55, 619, 1095, 1095, 1095, 1095, + 1095, 1095, 1095, 601, 600, 599, 598, 1096, 597, 55, + 55, 596, 55, 1096, 1096, 1096, 1096, 1096, 1096, 53, + 1096, 1096, 1096, 1096, 1096, 1096, 1096, 55, 267, 1097, + 1097, 1097, 1097, 1097, 1097, 1097, 1248, 1249, 1249, 1249, + 1249, 1249, 1249, 55, 55, 55, 910, 1100, 1100, 1100, + + 1100, 1100, 1100, 1100, 1101, 595, 594, 593, 1102, 592, + 914, 55, 55, 591, 1102, 1102, 1102, 1102, 1102, 1102, + 1252, 1253, 1253, 1253, 1253, 1253, 1253, 1259, 1260, 1260, + 1260, 1260, 1260, 1260, 55, 1710, 914, 919, 1103, 1103, + 1103, 1103, 1103, 1103, 1103, 1104, 590, 589, 581, 1105, + 578, 923, 55, 575, 574, 1105, 1105, 1105, 1105, 1105, + 1105, 1262, 1263, 1263, 1263, 1263, 1263, 1263, 1266, 1267, + 1267, 1267, 1267, 1267, 1267, 55, 1714, 923, 930, 1106, + 1106, 1106, 1106, 1106, 1106, 1106, 1107, 573, 572, 571, + 1108, 570, 934, 55, 569, 568, 1108, 1108, 1108, 1108, + + 1108, 1108, 1273, 1274, 1274, 1274, 1274, 1274, 1274, 1276, + 1277, 1277, 1277, 1277, 1277, 1277, 55, 55, 934, 942, + 1109, 1109, 1109, 1109, 1109, 1109, 1109, 1110, 567, 566, + 470, 1111, 279, 946, 55, 55, 453, 1111, 1111, 1111, + 1111, 1111, 1111, 1280, 1281, 1281, 1281, 1281, 1281, 1281, + 1168, 1168, 1168, 1168, 1168, 1168, 1168, 365, 551, 946, + 1113, 1114, 1114, 1114, 1114, 1114, 1114, 1115, 535, 542, + 453, 1116, 539, 450, 444, 55, 55, 1116, 1116, 1116, + 1116, 1116, 1116, 1117, 1118, 1118, 1118, 1118, 1118, 1118, + 1118, 1119, 55, 55, 55, 1120, 55, 1121, 55, 1350, + + 55, 1120, 1120, 1120, 1120, 1120, 1120, 1308, 1305, 535, + 55, 533, 444, 1728, 55, 436, 55, 55, 55, 430, + 520, 1303, 519, 1121, 910, 1124, 1124, 1124, 1124, 1124, + 1124, 1124, 1101, 1312, 1583, 55, 1125, 1402, 1717, 1728, + 55, 55, 1125, 1125, 1125, 1125, 1125, 1125, 1126, 1127, + 1127, 1127, 1127, 1127, 1127, 1127, 1128, 55, 55, 55, + 1129, 518, 1130, 517, 516, 55, 1129, 1129, 1129, 1129, + 1129, 1129, 1309, 1313, 515, 55, 1337, 1338, 1338, 1338, + 1338, 1338, 1338, 55, 1352, 55, 1307, 514, 1130, 919, + 1135, 1135, 1135, 1135, 1135, 1135, 1135, 1104, 1353, 513, + + 1403, 1136, 512, 55, 511, 55, 510, 1136, 1136, 1136, + 1136, 1136, 1136, 1137, 1138, 1138, 1138, 1138, 1138, 1138, + 1138, 1139, 1405, 55, 1353, 1140, 509, 1141, 1352, 508, + 507, 1140, 1140, 1140, 1140, 1140, 1140, 1317, 506, 503, + 502, 499, 1728, 1246, 1246, 1246, 1246, 1246, 1246, 1246, + 498, 1358, 55, 1141, 930, 1147, 1147, 1147, 1147, 1147, + 1147, 1147, 1107, 497, 496, 1359, 1148, 495, 1728, 120, + 55, 494, 1148, 1148, 1148, 1148, 1148, 1148, 1149, 1150, + 1150, 1150, 1150, 1150, 1150, 1150, 1151, 1408, 1358, 1360, + 1152, 1359, 1153, 1360, 493, 481, 1152, 1152, 1152, 1152, + + 1152, 1152, 1728, 1361, 480, 479, 478, 1728, 1260, 1260, + 1260, 1260, 1260, 1260, 1260, 477, 1366, 55, 1153, 942, + 1159, 1159, 1159, 1159, 1159, 1159, 1159, 1110, 1728, 1361, + 1367, 1160, 476, 1728, 475, 55, 474, 1160, 1160, 1160, + 1160, 1160, 1160, 1161, 1162, 1162, 1162, 1162, 1162, 1162, + 1162, 1163, 1475, 1366, 1368, 1164, 1367, 1165, 1368, 473, + 277, 1164, 1164, 1164, 1164, 1164, 1164, 1728, 1369, 470, + 279, 453, 1728, 1274, 1274, 1274, 1274, 1274, 1274, 1274, + 367, 1374, 55, 1165, 1068, 1176, 1176, 1176, 1176, 1176, + 1176, 1176, 1070, 1728, 1369, 1375, 1177, 450, 1728, 444, + + 55, 1374, 1177, 1177, 1177, 1177, 1177, 1177, 1178, 1179, + 1179, 1179, 1179, 1179, 1179, 1728, 444, 1547, 437, 1180, + 437, 1375, 55, 429, 55, 1180, 1180, 1180, 1180, 1180, + 1180, 55, 619, 1204, 1204, 1204, 1204, 1204, 1204, 1204, + 55, 1728, 55, 420, 989, 1715, 419, 1719, 416, 55, + 989, 989, 989, 989, 989, 989, 53, 989, 989, 989, + 989, 989, 989, 989, 55, 267, 1097, 1097, 1097, 1097, + 1097, 1097, 1097, 415, 55, 55, 1376, 1376, 55, 55, + 414, 1421, 55, 1207, 1208, 1208, 1208, 1208, 1208, 1208, + 1377, 1728, 55, 55, 1209, 1422, 55, 55, 1399, 413, + + 1209, 1209, 1209, 1209, 1209, 1209, 910, 1012, 1012, 1012, + 1012, 1012, 1012, 1012, 1101, 1400, 1377, 1728, 1013, 412, + 411, 1422, 410, 1421, 1013, 1013, 1013, 1013, 1013, 1013, + 1210, 1211, 1211, 1211, 1211, 1211, 1211, 1728, 409, 408, + 407, 1212, 406, 405, 404, 403, 55, 1212, 1212, 1212, + 1212, 1212, 1212, 919, 1024, 1024, 1024, 1024, 1024, 1024, + 1024, 1104, 402, 1728, 55, 1025, 401, 400, 395, 394, + 1468, 1025, 1025, 1025, 1025, 1025, 1025, 1213, 1214, 1214, + 1214, 1214, 1214, 1214, 1469, 1698, 393, 392, 1215, 391, + 320, 390, 389, 55, 1215, 1215, 1215, 1215, 1215, 1215, + + 930, 1039, 1039, 1039, 1039, 1039, 1039, 1039, 1107, 388, + 1469, 55, 1040, 387, 386, 385, 379, 1468, 1040, 1040, + 1040, 1040, 1040, 1040, 1216, 1217, 1217, 1217, 1217, 1217, + 1217, 1728, 1723, 279, 367, 1218, 275, 364, 272, 360, + 266, 1218, 1218, 1218, 1218, 1218, 1218, 942, 1055, 1055, + 1055, 1055, 1055, 1055, 1055, 1110, 260, 1728, 143, 1056, + 348, 347, 346, 341, 1470, 1056, 1056, 1056, 1056, 1056, + 1056, 1219, 1220, 1220, 1220, 1220, 1220, 1220, 1471, 320, + 340, 339, 1221, 338, 337, 336, 335, 334, 1221, 1221, + 1221, 1221, 1221, 1221, 1223, 1224, 1224, 1224, 1224, 1224, + + 1224, 1224, 1225, 333, 1471, 332, 1226, 331, 1227, 330, + 329, 328, 1226, 1226, 1226, 1226, 1226, 1226, 1382, 1383, + 1383, 1383, 1383, 1383, 1383, 1385, 1386, 1386, 1386, 1386, + 1386, 1386, 55, 327, 1227, 1117, 1231, 1231, 1231, 1231, + 1231, 1231, 1231, 1119, 326, 323, 322, 1232, 321, 320, + 55, 1406, 319, 1232, 1232, 1232, 1232, 1232, 1232, 1126, + 1241, 1241, 1241, 1241, 1241, 1241, 1241, 1128, 1407, 318, + 317, 1242, 316, 315, 314, 313, 312, 1242, 1242, 1242, + 1242, 1242, 1242, 1137, 1254, 1254, 1254, 1254, 1254, 1254, + 1254, 1139, 311, 310, 309, 1255, 308, 307, 306, 305, + + 304, 1255, 1255, 1255, 1255, 1255, 1255, 1149, 1268, 1268, + 1268, 1268, 1268, 1268, 1268, 1151, 303, 302, 301, 1269, + 300, 299, 298, 297, 143, 1269, 1269, 1269, 1269, 1269, + 1269, 1161, 1282, 1282, 1282, 1282, 1282, 1282, 1282, 1163, + 296, 295, 294, 1283, 293, 292, 291, 290, 289, 1283, + 1283, 1283, 1283, 1283, 1283, 1068, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1070, 288, 287, 286, 1292, 285, 284, + 283, 282, 279, 1292, 1292, 1292, 1292, 1292, 1292, 1293, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1295, 267, 272, + 259, 1296, 258, 1297, 55, 55, 1470, 1296, 1296, 1296, + + 1296, 1296, 1296, 1389, 1390, 1390, 1390, 1390, 1390, 1390, + 1728, 55, 55, 55, 257, 55, 55, 55, 256, 1297, + 55, 619, 525, 525, 525, 525, 525, 525, 525, 55, + 255, 1534, 1576, 55, 55, 55, 1728, 1506, 55, 1318, + 1319, 1319, 1319, 1319, 1319, 1319, 1320, 1397, 1410, 1660, + 1321, 1507, 1586, 254, 253, 55, 1321, 1321, 1321, 1321, + 1321, 1321, 1117, 1322, 1322, 1322, 1322, 1322, 1322, 1322, + 1323, 55, 55, 55, 1324, 252, 1121, 1507, 251, 250, + 1324, 1324, 1324, 1324, 1324, 1324, 247, 1480, 244, 55, + 55, 1427, 1428, 1428, 1428, 1428, 1428, 1428, 241, 240, + + 1398, 1411, 1121, 1126, 1325, 1325, 1325, 1325, 1325, 1325, + 1325, 1326, 239, 238, 235, 1327, 234, 1130, 233, 232, + 231, 1327, 1327, 1327, 1327, 1327, 1327, 1432, 1433, 1433, + 1433, 1433, 1433, 1433, 1435, 1436, 1436, 1436, 1436, 1436, + 1436, 230, 229, 1130, 1137, 1328, 1328, 1328, 1328, 1328, + 1328, 1328, 1329, 228, 227, 226, 1330, 225, 1141, 224, + 223, 222, 1330, 1330, 1330, 1330, 1330, 1330, 1441, 1442, + 1442, 1442, 1442, 1442, 1442, 1444, 1445, 1445, 1445, 1445, + 1445, 1445, 221, 220, 1141, 1149, 1331, 1331, 1331, 1331, + 1331, 1331, 1331, 1332, 216, 215, 205, 1333, 204, 1153, + + 203, 200, 199, 1333, 1333, 1333, 1333, 1333, 1333, 1450, + 1451, 1451, 1451, 1451, 1451, 1451, 1453, 1454, 1454, 1454, + 1454, 1454, 1454, 194, 193, 1153, 1161, 1334, 1334, 1334, + 1334, 1334, 1334, 1334, 1335, 192, 191, 190, 1336, 189, + 1165, 188, 187, 186, 1336, 1336, 1336, 1336, 1336, 1336, + 1459, 1460, 1460, 1460, 1460, 1460, 1460, 1462, 1463, 1463, + 1463, 1463, 1463, 1463, 1506, 55, 1165, 1223, 1339, 1339, + 1339, 1339, 1339, 1339, 1339, 1225, 185, 181, 1728, 1340, + 180, 179, 178, 55, 177, 1340, 1340, 1340, 1340, 1340, + 1340, 1117, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1323, + + 176, 1579, 175, 1348, 1728, 174, 173, 172, 171, 1348, + 1348, 1348, 1348, 1348, 1348, 1126, 1354, 1354, 1354, 1354, + 1354, 1354, 1354, 1326, 170, 169, 168, 1355, 167, 166, + 165, 160, 151, 1355, 1355, 1355, 1355, 1355, 1355, 1137, + 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1329, 150, 145, + 59, 1363, 47, 45, 1728, 1728, 1728, 1363, 1363, 1363, + 1363, 1363, 1363, 1149, 1370, 1370, 1370, 1370, 1370, 1370, + 1370, 1332, 1728, 1728, 1728, 1371, 1728, 1728, 1728, 1728, + 1728, 1371, 1371, 1371, 1371, 1371, 1371, 1161, 1378, 1378, + 1378, 1378, 1378, 1378, 1378, 1335, 1728, 1728, 1728, 1379, + + 1728, 1728, 1728, 1728, 1728, 1379, 1379, 1379, 1379, 1379, + 1379, 1293, 1391, 1391, 1391, 1391, 1391, 1391, 1391, 1295, + 1728, 1728, 1728, 1392, 1728, 1728, 1728, 1728, 1728, 1392, + 1392, 1392, 1392, 1392, 1392, 1223, 1412, 1412, 1412, 1412, + 1412, 1412, 1412, 1413, 1728, 1728, 1728, 1414, 1728, 1227, + 1728, 1728, 1728, 1414, 1414, 1414, 1414, 1414, 1414, 1383, + 1383, 1383, 1383, 1383, 1383, 1383, 1500, 1501, 1501, 1501, + 1501, 1501, 1501, 1510, 55, 1227, 1117, 1231, 1231, 1231, + 1231, 1231, 1231, 1231, 1323, 1728, 1728, 1511, 1232, 1728, + 1728, 1728, 55, 1728, 1232, 1232, 1232, 1232, 1232, 1232, + + 1126, 1241, 1241, 1241, 1241, 1241, 1241, 1241, 1326, 1728, + 1580, 1728, 1242, 1511, 1728, 1728, 1728, 1728, 1242, 1242, + 1242, 1242, 1242, 1242, 1137, 1254, 1254, 1254, 1254, 1254, + 1254, 1254, 1329, 1728, 1728, 1728, 1255, 1728, 1728, 1728, + 1728, 1728, 1255, 1255, 1255, 1255, 1255, 1255, 1149, 1268, + 1268, 1268, 1268, 1268, 1268, 1268, 1332, 1728, 1728, 1728, + 1269, 1728, 1728, 1728, 1728, 1728, 1269, 1269, 1269, 1269, + 1269, 1269, 1161, 1282, 1282, 1282, 1282, 1282, 1282, 1282, + 1335, 1728, 1728, 1728, 1283, 1728, 1728, 1728, 1728, 1728, + 1283, 1283, 1283, 1283, 1283, 1283, 1223, 1423, 1423, 1423, + + 1423, 1423, 1423, 1423, 1413, 1728, 1728, 1728, 1424, 1728, + 1728, 1728, 55, 1728, 1424, 1424, 1424, 1424, 1424, 1424, + 1293, 1472, 1472, 1472, 1472, 1472, 1472, 1472, 1295, 55, + 55, 55, 1473, 1728, 1510, 1728, 1728, 1728, 1473, 1473, + 1473, 1473, 1473, 1473, 1487, 55, 1728, 55, 1728, 55, + 1433, 1433, 1433, 1433, 1433, 1433, 1433, 1728, 1484, 1728, + 1728, 1488, 1514, 55, 1485, 1223, 1339, 1339, 1339, 1339, + 1339, 1339, 1339, 1413, 1728, 1548, 1515, 1340, 1728, 1728, + 1728, 1728, 1514, 1340, 1340, 1340, 1340, 1340, 1340, 1442, + 1442, 1442, 1442, 1442, 1442, 1442, 1728, 1518, 1518, 1728, + + 1728, 1728, 1515, 1451, 1451, 1451, 1451, 1451, 1451, 1451, + 1522, 1519, 1728, 1460, 1460, 1460, 1460, 1460, 1460, 1460, + 1522, 1728, 1728, 1728, 1523, 1526, 1527, 1527, 1527, 1527, + 1527, 1527, 1728, 55, 1728, 55, 55, 1519, 1728, 1529, + 1530, 1530, 1530, 1530, 1530, 1530, 55, 55, 55, 1728, + 1523, 55, 55, 55, 55, 1551, 1728, 1551, 55, 55, + 1728, 55, 1538, 1539, 55, 55, 55, 1543, 1728, 1552, + 55, 1728, 1728, 1540, 1541, 1542, 55, 55, 1728, 55, + 1728, 1545, 1553, 1554, 1554, 1554, 1554, 1554, 1554, 1728, + 1728, 1728, 1728, 1622, 1582, 1552, 1587, 1728, 1557, 1558, + + 1558, 1558, 1558, 1558, 1558, 1561, 1562, 1562, 1562, 1562, + 1562, 1562, 1565, 1566, 1566, 1566, 1566, 1566, 1566, 1569, + 1570, 1570, 1570, 1570, 1570, 1570, 1527, 1527, 1527, 1527, + 1527, 1527, 1527, 1573, 1573, 55, 55, 55, 1589, 1590, + 1590, 1590, 1590, 1590, 1590, 1728, 1728, 1574, 1728, 1728, + 1728, 1728, 1728, 55, 55, 55, 1728, 1728, 1728, 1728, + 1728, 1728, 1578, 1585, 1588, 1554, 1554, 1554, 1554, 1554, + 1554, 1554, 1728, 1574, 1728, 1558, 1558, 1558, 1558, 1558, + 1558, 1558, 1562, 1562, 1562, 1562, 1562, 1562, 1562, 1566, + 1566, 1566, 1566, 1566, 1566, 1566, 1570, 1570, 1570, 1570, + + 1570, 1570, 1570, 1598, 1599, 1599, 1599, 1599, 1599, 1599, + 55, 55, 1728, 55, 55, 55, 55, 55, 1590, 1590, + 1590, 1590, 1590, 1590, 1590, 55, 55, 1728, 55, 55, + 1605, 55, 55, 55, 55, 55, 1728, 1728, 1728, 1728, + 1728, 1728, 1611, 55, 55, 1601, 55, 1606, 55, 55, + 1607, 1608, 1609, 1612, 1599, 1599, 1599, 1599, 1599, 1599, + 1599, 55, 1620, 1628, 55, 55, 55, 55, 55, 55, + 1728, 55, 55, 1621, 55, 55, 55, 55, 1728, 55, + 1623, 1625, 55, 55, 1728, 1728, 55, 55, 1629, 55, + 55, 55, 55, 55, 55, 55, 1647, 1630, 1637, 1728, + + 55, 55, 1638, 1645, 55, 1633, 1641, 1639, 1728, 55, + 55, 1640, 55, 1648, 1649, 55, 1650, 1651, 1728, 55, + 55, 1728, 55, 55, 55, 55, 1652, 1728, 55, 1728, + 55, 55, 55, 55, 1728, 55, 1653, 55, 55, 1654, + 1728, 55, 55, 55, 1658, 1656, 1666, 1665, 1728, 55, + 55, 55, 1664, 55, 1670, 55, 1675, 55, 1667, 1668, + 1677, 55, 55, 55, 55, 1728, 1671, 55, 1673, 55, + 1674, 55, 55, 1676, 1728, 55, 55, 1728, 55, 55, + 55, 55, 55, 1678, 1728, 55, 55, 1728, 1682, 55, + 55, 55, 1683, 1728, 55, 1679, 55, 1685, 1684, 1688, + + 55, 1728, 1689, 55, 55, 1691, 55, 55, 1728, 55, + 1695, 1692, 55, 55, 55, 55, 1728, 1728, 55, 1696, + 1697, 55, 55, 1728, 55, 55, 1701, 55, 55, 1699, + 55, 55, 55, 55, 1704, 1702, 55, 55, 55, 1728, + 55, 1706, 1709, 1703, 1705, 55, 55, 55, 55, 55, + 55, 1728, 1707, 1728, 55, 55, 55, 1711, 1728, 1728, + 1728, 1728, 1728, 1716, 1712, 55, 55, 55, 55, 1728, + 1728, 1728, 1713, 1728, 1718, 1721, 1722, 55, 1728, 1728, + 1726, 1727, 1728, 1720, 1728, 1724, 1728, 1725, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 55, 46, 1728, 1728, 1728, + + 1728, 46, 46, 46, 64, 1728, 64, 64, 64, 64, + 64, 64, 64, 152, 1728, 152, 159, 159, 159, 271, + 271, 271, 280, 280, 280, 359, 359, 359, 362, 362, + 362, 363, 363, 363, 370, 370, 370, 368, 368, 368, + 374, 374, 374, 378, 1728, 378, 443, 443, 443, 448, + 448, 448, 449, 449, 449, 458, 458, 458, 462, 1728, + 462, 463, 463, 463, 372, 372, 1728, 1728, 372, 467, + 467, 467, 471, 471, 471, 362, 362, 362, 532, 532, + 532, 536, 536, 536, 537, 537, 537, 538, 538, 538, + 370, 370, 370, 543, 543, 543, 456, 456, 1728, 1728, + + 456, 548, 548, 548, 552, 552, 552, 556, 1728, 556, + 557, 557, 557, 561, 561, 561, 565, 1728, 565, 624, + 624, 624, 458, 458, 458, 632, 632, 632, 633, 633, + 633, 641, 641, 641, 645, 1728, 645, 648, 1728, 648, + 649, 649, 649, 653, 653, 653, 657, 1728, 657, 555, + 555, 1728, 1728, 555, 559, 559, 1728, 1728, 559, 663, + 663, 663, 667, 667, 667, 565, 565, 1728, 565, 537, + 537, 537, 713, 713, 713, 717, 717, 717, 720, 720, + 720, 721, 721, 721, 722, 722, 722, 727, 727, 727, + 639, 639, 1728, 1728, 639, 732, 732, 732, 736, 736, + + 736, 645, 645, 1728, 645, 647, 647, 1728, 1728, 647, + 648, 648, 1728, 648, 649, 649, 651, 651, 1728, 1728, + 651, 743, 743, 743, 747, 747, 747, 657, 657, 1728, + 657, 751, 1728, 751, 754, 1728, 754, 755, 755, 755, + 759, 759, 759, 763, 1728, 763, 802, 802, 802, 641, + 641, 641, 653, 653, 653, 813, 813, 813, 814, 814, + 814, 822, 822, 822, 826, 1728, 826, 829, 1728, 829, + 830, 830, 830, 834, 834, 834, 838, 1728, 838, 841, + 1728, 841, 844, 1728, 844, 845, 845, 845, 849, 849, + 849, 853, 1728, 853, 750, 1728, 1728, 750, 751, 751, + + 1728, 751, 753, 753, 1728, 1728, 753, 754, 754, 1728, + 754, 755, 755, 757, 757, 1728, 1728, 757, 860, 860, + 860, 864, 864, 864, 763, 763, 1728, 763, 53, 53, + 53, 1728, 53, 53, 721, 721, 721, 896, 896, 896, + 900, 900, 900, 903, 903, 903, 906, 906, 906, 907, + 907, 907, 908, 908, 908, 913, 913, 913, 820, 820, + 1728, 1728, 820, 918, 918, 918, 922, 922, 922, 826, + 826, 1728, 826, 828, 828, 1728, 1728, 828, 829, 829, + 1728, 829, 830, 830, 832, 832, 1728, 1728, 832, 929, + 929, 929, 933, 933, 933, 838, 838, 1728, 838, 840, + + 1728, 1728, 840, 841, 841, 1728, 841, 843, 843, 1728, + 1728, 843, 844, 844, 1728, 844, 845, 845, 847, 847, + 1728, 1728, 847, 941, 941, 941, 945, 945, 945, 853, + 853, 1728, 853, 947, 1728, 947, 950, 1728, 950, 953, + 1728, 953, 954, 954, 954, 958, 958, 958, 962, 1728, + 962, 53, 53, 53, 1728, 53, 53, 990, 990, 990, + 822, 822, 822, 834, 834, 834, 849, 849, 849, 1004, + 1004, 1004, 1005, 1005, 1005, 1013, 1013, 1013, 1017, 1728, + 1017, 1020, 1728, 1020, 1021, 1021, 1021, 1025, 1025, 1025, + 1029, 1728, 1029, 1032, 1728, 1032, 1035, 1728, 1035, 1036, + + 1036, 1036, 1040, 1040, 1040, 1044, 1728, 1044, 1045, 1728, + 1045, 1048, 1728, 1048, 1051, 1728, 1051, 1052, 1052, 1052, + 1056, 1056, 1056, 1060, 1728, 1060, 947, 1728, 947, 949, + 1728, 1728, 949, 950, 950, 1728, 950, 952, 952, 1728, + 1728, 952, 953, 953, 1728, 953, 954, 954, 956, 956, + 1728, 1728, 956, 1067, 1067, 1067, 1071, 1071, 1071, 962, + 962, 1728, 962, 53, 53, 53, 1728, 53, 53, 907, + 907, 907, 1098, 1098, 1098, 1102, 1102, 1102, 1105, 1105, + 1105, 1108, 1108, 1108, 1111, 1111, 1111, 1112, 1112, 1112, + 1120, 1120, 1120, 1011, 1011, 1728, 1728, 1011, 1125, 1125, + + 1125, 1129, 1129, 1129, 1017, 1017, 1728, 1017, 1019, 1019, + 1728, 1728, 1019, 1020, 1020, 1728, 1020, 1021, 1021, 1023, + 1023, 1728, 1728, 1023, 1136, 1136, 1136, 1140, 1140, 1140, + 1029, 1029, 1728, 1029, 1031, 1728, 1728, 1031, 1032, 1032, + 1728, 1032, 1034, 1034, 1728, 1728, 1034, 1035, 1035, 1728, + 1035, 1036, 1036, 1038, 1038, 1728, 1728, 1038, 1148, 1148, + 1148, 1152, 1152, 1152, 1044, 1044, 1728, 1044, 1045, 1728, + 1045, 1047, 1728, 1728, 1047, 1048, 1048, 1728, 1048, 1050, + 1050, 1728, 1728, 1050, 1051, 1051, 1728, 1051, 1052, 1052, + 1054, 1054, 1728, 1728, 1054, 1160, 1160, 1160, 1164, 1164, + + 1164, 1060, 1060, 1728, 1060, 1166, 1728, 1166, 1169, 1728, + 1169, 1172, 1728, 1172, 1173, 1173, 1173, 1177, 1177, 1177, + 1181, 1728, 1181, 53, 53, 53, 1728, 53, 53, 1205, + 1205, 1205, 1013, 1013, 1013, 1025, 1025, 1025, 1040, 1040, + 1040, 1056, 1056, 1056, 1222, 1222, 1222, 1228, 1228, 1228, + 1226, 1226, 1226, 1233, 1233, 1233, 1232, 1232, 1232, 1234, + 1728, 1234, 1237, 1728, 1237, 1238, 1238, 1238, 1243, 1243, + 1243, 1242, 1242, 1242, 1244, 1728, 1244, 1247, 1728, 1247, + 1250, 1728, 1250, 1251, 1251, 1251, 1256, 1256, 1256, 1255, + 1255, 1255, 1257, 1728, 1257, 1258, 1728, 1258, 1261, 1728, + + 1261, 1264, 1728, 1264, 1265, 1265, 1265, 1270, 1270, 1270, + 1269, 1269, 1269, 1271, 1728, 1271, 1272, 1728, 1272, 1275, + 1728, 1275, 1278, 1728, 1278, 1279, 1279, 1279, 1284, 1284, + 1284, 1283, 1283, 1283, 1285, 1728, 1285, 1166, 1728, 1166, + 1168, 1728, 1728, 1168, 1169, 1169, 1728, 1169, 1171, 1171, + 1728, 1728, 1171, 1172, 1172, 1728, 1172, 1173, 1173, 1175, + 1175, 1728, 1728, 1175, 1292, 1292, 1292, 1296, 1296, 1296, + 1181, 1181, 1728, 1181, 53, 53, 53, 1728, 53, 53, + 1112, 1112, 1112, 1324, 1324, 1324, 1327, 1327, 1327, 1330, + 1330, 1330, 1333, 1333, 1333, 1336, 1336, 1336, 1341, 1341, + + 1341, 1340, 1340, 1340, 1343, 1728, 1343, 1344, 1344, 1344, + 1230, 1230, 1728, 1728, 1230, 1348, 1348, 1348, 1349, 1349, + 1349, 1234, 1234, 1728, 1234, 1236, 1236, 1728, 1728, 1236, + 1237, 1237, 1728, 1237, 1238, 1238, 1240, 1240, 1728, 1728, + 1240, 1355, 1355, 1355, 1356, 1356, 1356, 1244, 1244, 1728, + 1244, 1246, 1728, 1728, 1246, 1247, 1247, 1728, 1247, 1249, + 1249, 1728, 1728, 1249, 1250, 1250, 1728, 1250, 1251, 1251, + 1253, 1253, 1728, 1728, 1253, 1363, 1363, 1363, 1364, 1364, + 1364, 1257, 1257, 1728, 1257, 1258, 1728, 1258, 1260, 1728, + 1728, 1260, 1261, 1261, 1728, 1261, 1263, 1263, 1728, 1728, + + 1263, 1264, 1264, 1728, 1264, 1265, 1265, 1267, 1267, 1728, + 1728, 1267, 1371, 1371, 1371, 1372, 1372, 1372, 1271, 1271, + 1728, 1271, 1272, 1728, 1272, 1274, 1728, 1728, 1274, 1275, + 1275, 1728, 1275, 1277, 1277, 1728, 1728, 1277, 1278, 1278, + 1728, 1278, 1279, 1279, 1281, 1281, 1728, 1728, 1281, 1379, + 1379, 1379, 1380, 1380, 1380, 1285, 1285, 1728, 1285, 1381, + 1728, 1381, 1384, 1728, 1384, 1387, 1728, 1387, 1388, 1388, + 1388, 1393, 1728, 1393, 1392, 1392, 1392, 1394, 1728, 1394, + 53, 53, 53, 1728, 53, 53, 1415, 1728, 1415, 1414, + 1414, 1414, 1416, 1728, 1416, 1232, 1232, 1232, 1417, 1728, + + 1417, 1242, 1242, 1242, 1418, 1728, 1418, 1255, 1255, 1255, + 1419, 1728, 1419, 1269, 1269, 1269, 1420, 1728, 1420, 1283, + 1283, 1283, 1338, 1338, 1728, 1728, 1338, 1424, 1424, 1424, + 1425, 1425, 1425, 370, 370, 370, 1343, 1343, 1728, 1343, + 1426, 1426, 1426, 1429, 1728, 1429, 1430, 1430, 1430, 1431, + 1431, 1431, 1434, 1728, 1434, 1437, 1728, 1437, 1438, 1438, + 1438, 1439, 1439, 1439, 1440, 1728, 1440, 1443, 1728, 1443, + 1446, 1728, 1446, 1447, 1447, 1447, 1448, 1448, 1448, 1449, + 1728, 1449, 1452, 1728, 1452, 1455, 1728, 1455, 1456, 1456, + 1456, 1457, 1457, 1457, 1458, 1728, 1458, 1461, 1728, 1461, + + 1464, 1728, 1464, 1465, 1465, 1465, 1466, 1466, 1466, 1381, + 1728, 1381, 1383, 1728, 1728, 1383, 1384, 1384, 1728, 1384, + 1386, 1386, 1728, 1728, 1386, 1387, 1387, 1728, 1387, 1388, + 1388, 1390, 1390, 1728, 1728, 1390, 1473, 1473, 1473, 1474, + 1728, 1474, 1394, 1394, 1728, 1394, 53, 53, 53, 1728, + 53, 53, 1492, 1492, 1492, 1340, 1340, 1340, 1494, 1728, + 1494, 1495, 1728, 1495, 1496, 1728, 1496, 1497, 1728, 1497, + 1498, 1728, 1498, 1499, 1728, 1499, 1502, 1728, 1502, 1503, + 1503, 1503, 1504, 1504, 1504, 1505, 1728, 1505, 1428, 1428, + 1728, 1728, 1428, 1429, 1429, 1728, 1429, 1430, 1430, 1508, + + 1728, 1508, 1433, 1728, 1728, 1433, 1434, 1434, 1728, 1434, + 1436, 1436, 1728, 1728, 1436, 1437, 1437, 1728, 1437, 1438, + 1438, 1512, 1728, 1512, 1440, 1728, 1440, 1442, 1728, 1728, + 1442, 1443, 1443, 1728, 1443, 1445, 1445, 1728, 1728, 1445, + 1446, 1446, 1728, 1446, 1447, 1447, 1516, 1728, 1516, 1449, + 1728, 1449, 1451, 1728, 1728, 1451, 1452, 1452, 1728, 1452, + 1454, 1454, 1728, 1728, 1454, 1455, 1455, 1728, 1455, 1456, + 1456, 1520, 1728, 1520, 1458, 1728, 1458, 1460, 1728, 1728, + 1460, 1461, 1461, 1728, 1461, 1463, 1463, 1728, 1728, 1463, + 1464, 1464, 1728, 1464, 1465, 1465, 1524, 1728, 1524, 1525, + + 1728, 1525, 1528, 1728, 1528, 1531, 1728, 1531, 1532, 1532, + 1532, 1533, 1728, 1533, 53, 53, 53, 1728, 53, 53, + 1550, 1728, 1550, 1426, 1728, 1426, 1431, 1728, 1431, 1439, + 1728, 1439, 1448, 1728, 1448, 1457, 1728, 1457, 1466, 1728, + 1466, 1501, 1501, 1728, 1728, 1501, 1502, 1502, 1728, 1502, + 1503, 1503, 1493, 1728, 1493, 1555, 1728, 1555, 1556, 1728, + 1556, 1559, 1728, 1559, 1560, 1728, 1560, 1563, 1728, 1563, + 1564, 1728, 1564, 1567, 1728, 1567, 1568, 1728, 1568, 1571, + 1728, 1571, 1527, 1728, 1728, 1527, 1530, 1530, 1728, 1728, + 1530, 1575, 1728, 1575, 1504, 1728, 1504, 1591, 1728, 1591, + + 1554, 1728, 1728, 1554, 1558, 1728, 1728, 1558, 1562, 1728, + 1728, 1562, 1566, 1728, 1728, 1566, 1570, 1728, 1728, 1570, + 1597, 1728, 1597, 1600, 1728, 1600, 1590, 1728, 1728, 1590, + 1614, 1728, 1614, 1615, 1728, 1615, 1616, 1728, 1616, 1617, + 1728, 1617, 1618, 1728, 1618, 1599, 1728, 1728, 1599, 1631, + 1728, 1631, 1632, 1728, 1632, 3, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728 } ; -static yyconst flex_int16_t yy_chk[7486] = +static yyconst flex_int16_t yy_chk[7910] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -1855,820 +2009,867 @@ static yyconst flex_int16_t yy_chk[7486] = 7, 106, 7, 7, 9, 17, 17, 9, 138, 9, 9, 4, 19, 19, 24, 7, 27, 27, 138, 4, 15, 10, 9, 7, 10, 102, 10, 10, 24, 28, - 9, 34, 26, 1234, 28, 24, 26, 659, 94, 10, + 9, 34, 26, 1278, 28, 24, 26, 586, 94, 10, - 34, 94, 26, 659, 95, 26, 95, 10, 14, 14, + 34, 94, 26, 586, 95, 26, 95, 10, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 102, 98, - 1237, 14, 14, 14, 98, 83, 14, 14, 14, 14, + 1279, 14, 14, 14, 98, 83, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, - 14, 16, 83, 25, 16, 1240, 16, 16, 83, 129, - 29, 1243, 107, 25, 29, 252, 25, 107, 22, 16, + 14, 16, 83, 25, 16, 671, 16, 16, 83, 129, + 29, 671, 107, 25, 29, 252, 25, 107, 22, 16, 25, 22, 129, 22, 22, 103, 29, 16, 20, 20, 20, 20, 20, 20, 20, 20, 22, 134, 23, 252, - 23, 23, 20, 661, 22, 23, 661, 251, 35, 251, + 23, 23, 20, 673, 22, 23, 673, 251, 35, 251, 23, 30, 103, 30, 23, 134, 23, 103, 30, 30, - 35, 30, 35, 1246, 35, 30, 1250, 35, 20, 21, + 35, 30, 35, 1283, 35, 30, 1284, 35, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 32, - 32, 110, 21, 663, 110, 1251, 110, 136, 21, 21, + 32, 110, 21, 675, 110, 1285, 110, 136, 21, 21, 21, 21, 21, 21, 31, 32, 136, 31, 31, 32, - 31, 663, 31, 32, 31, 32, 31, 33, 36, 31, - 155, 33, 32, 36, 36, 37, 36, 36, 1252, 37, + 31, 675, 31, 32, 31, 32, 31, 33, 36, 31, + 155, 33, 32, 36, 36, 37, 36, 36, 710, 37, 33, 33, 163, 38, 33, 222, 33, 37, 39, 38, 39, 38, 37, 38, 37, 41, 155, 38, 222, 38, 38, 267, 267, 41, 39, 437, 437, 41, 163, 41, - 44, 39, 666, 44, 666, 44, 44, 62, 62, 62, + 44, 39, 397, 44, 710, 44, 44, 62, 62, 62, 62, 62, 62, 62, 274, 339, 63, 351, 44, 63, - 339, 63, 63, 410, 1253, 413, 44, 48, 48, 48, - 48, 48, 48, 48, 63, 1254, 410, 667, 48, 413, - 274, 667, 63, 351, 48, 48, 48, 48, 48, 48, - 49, 49, 49, 49, 49, 49, 49, 376, 439, 1258, - 446, 49, 284, 284, 284, 284, 1259, 49, 49, 49, + 339, 63, 63, 397, 410, 1292, 44, 48, 48, 48, + 48, 48, 48, 48, 63, 619, 619, 410, 48, 397, + 274, 680, 63, 351, 48, 48, 48, 48, 48, 48, + 49, 49, 49, 49, 49, 49, 49, 376, 439, 680, + 446, 49, 284, 284, 284, 284, 1296, 49, 49, 49, 49, 49, 49, 50, 50, 50, 50, 50, 50, 50, 50, 466, 284, 376, 439, 50, 446, 452, 460, 466, - 1265, 50, 50, 50, 50, 50, 50, 52, 52, 52, + 1321, 50, 50, 50, 50, 50, 50, 52, 52, 52, 52, 52, 52, 52, 52, 54, 54, 54, 54, 54, - 54, 54, 486, 452, 460, 355, 54, 563, 563, 563, - 563, 486, 54, 54, 54, 54, 54, 54, 56, 355, - 1266, 56, 1273, 56, 56, 607, 607, 371, 372, 56, - 56, 56, 56, 56, 56, 56, 56, 520, 560, 1274, - 56, 371, 372, 1281, 56, 355, 56, 56, 56, 56, + 54, 54, 486, 452, 460, 355, 54, 1324, 413, 670, + 670, 486, 54, 54, 54, 54, 54, 54, 56, 355, + 672, 56, 413, 56, 56, 672, 1327, 371, 372, 56, + 56, 56, 56, 56, 56, 56, 56, 523, 563, 679, + 56, 371, 372, 679, 56, 355, 56, 56, 56, 56, 56, 56, 153, 488, 153, 153, 153, 153, 153, 153, - 153, 544, 488, 520, 560, 153, 615, 371, 372, 544, - 1282, 153, 153, 153, 153, 153, 153, 154, 154, 154, - 154, 154, 154, 154, 154, 554, 650, 658, 658, 154, + 153, 547, 488, 523, 563, 153, 627, 371, 372, 547, + 686, 153, 153, 153, 153, 153, 153, 154, 154, 154, + 154, 154, 154, 154, 154, 557, 662, 686, 1330, 154, - 618, 154, 615, 554, 650, 154, 154, 154, 154, 154, + 630, 154, 627, 557, 662, 154, 154, 154, 154, 154, 154, 266, 266, 266, 266, 266, 266, 266, 277, 277, - 277, 277, 277, 277, 277, 455, 618, 154, 156, 156, - 156, 156, 156, 156, 156, 156, 624, 631, 660, 455, - 156, 643, 684, 660, 677, 677, 156, 156, 156, 156, + 277, 277, 277, 277, 277, 455, 630, 154, 156, 156, + 156, 156, 156, 156, 156, 156, 587, 636, 1333, 455, + 156, 566, 566, 566, 566, 587, 156, 156, 156, 156, 156, 156, 157, 157, 157, 157, 157, 157, 157, 157, - 157, 686, 624, 631, 157, 455, 1289, 643, 684, 1290, + 157, 684, 1336, 636, 157, 455, 643, 655, 684, 1340, 157, 157, 157, 157, 157, 157, 158, 158, 158, 158, 158, 158, 158, 158, 160, 160, 160, 160, 160, 160, - 160, 160, 707, 1294, 725, 160, 737, 686, 668, 740, - - 707, 160, 160, 160, 160, 160, 160, 162, 162, 162, - 162, 162, 162, 162, 162, 162, 668, 740, 1297, 162, - 725, 162, 737, 309, 742, 162, 162, 162, 162, 162, - 162, 309, 361, 361, 361, 361, 361, 361, 361, 1298, - 754, 456, 742, 524, 309, 743, 664, 162, 260, 260, - 260, 260, 260, 260, 260, 456, 664, 524, 754, 260, - 309, 741, 1302, 743, 1304, 260, 260, 260, 260, 260, - 260, 262, 262, 262, 262, 262, 262, 262, 662, 741, - 743, 456, 262, 524, 769, 772, 741, 662, 262, 262, + 160, 160, 643, 655, 678, 160, 678, 682, 1341, 682, + + 1342, 160, 160, 160, 160, 160, 160, 162, 162, 162, + 162, 162, 162, 162, 162, 162, 687, 691, 691, 162, + 1343, 162, 1344, 309, 687, 162, 162, 162, 162, 162, + 162, 309, 361, 361, 361, 361, 361, 361, 361, 701, + 701, 456, 1348, 527, 309, 764, 676, 162, 260, 260, + 260, 260, 260, 260, 260, 456, 676, 527, 708, 260, + 309, 1349, 766, 764, 689, 260, 260, 260, 260, 260, + 260, 262, 262, 262, 262, 262, 262, 262, 588, 689, + 766, 456, 262, 527, 708, 749, 761, 588, 262, 262, 262, 262, 262, 262, 263, 263, 263, 263, 263, 263, - 263, 263, 265, 265, 265, 265, 265, 265, 265, 718, - 769, 772, 1310, 265, 775, 748, 752, 718, 749, 265, + 263, 263, 265, 265, 265, 265, 265, 265, 265, 674, + 805, 749, 761, 265, 1355, 1356, 1363, 772, 674, 265, 265, 265, 265, 265, 265, 268, 268, 268, 268, 268, - 268, 268, 268, 748, 752, 752, 749, 268, 748, 749, - 775, 781, 788, 268, 268, 268, 268, 268, 268, 270, + 268, 268, 268, 731, 742, 772, 805, 268, 1364, 1371, + 772, 731, 742, 268, 268, 268, 268, 268, 268, 270, 270, 270, 270, 270, 270, 270, 270, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 823, 781, 788, 273, - 1311, 273, 1321, 1322, 823, 273, 273, 273, 273, 273, + 273, 273, 273, 273, 273, 273, 859, 1372, 1379, 273, + 1380, 273, 1384, 1387, 859, 273, 273, 273, 273, 273, 273, 365, 365, 365, 365, 365, 365, 365, 436, 436, - 436, 436, 436, 436, 436, 551, 744, 273, 278, 278, - - 278, 278, 278, 278, 278, 278, 278, 1326, 1327, 551, - 278, 800, 755, 1331, 744, 669, 278, 278, 278, 278, - 278, 278, 279, 279, 279, 279, 279, 279, 279, 669, - 755, 744, 672, 279, 552, 551, 1334, 800, 1335, 279, - 279, 279, 279, 279, 279, 344, 672, 344, 552, 344, - 344, 464, 464, 464, 464, 464, 464, 464, 750, 344, - 1340, 344, 344, 555, 344, 349, 556, 349, 349, 349, - 349, 349, 349, 349, 552, 803, 750, 555, 349, 750, - 556, 759, 761, 761, 349, 349, 349, 349, 349, 349, - 350, 350, 350, 350, 350, 350, 350, 350, 1343, 759, - - 761, 803, 350, 555, 350, 1344, 556, 1349, 350, 350, + 436, 436, 436, 436, 436, 554, 683, 273, 278, 278, + + 278, 278, 278, 278, 278, 278, 278, 683, 1388, 554, + 278, 1392, 765, 779, 1394, 681, 278, 278, 278, 278, + 278, 278, 279, 279, 279, 279, 279, 279, 279, 681, + 765, 779, 685, 279, 555, 554, 779, 765, 1413, 279, + 279, 279, 279, 279, 279, 344, 685, 344, 555, 344, + 344, 464, 464, 464, 464, 464, 464, 464, 773, 344, + 690, 344, 344, 558, 344, 349, 559, 349, 349, 349, + 349, 349, 349, 349, 555, 690, 773, 558, 349, 773, + 559, 776, 1414, 787, 349, 349, 349, 349, 349, 349, + 350, 350, 350, 350, 350, 350, 350, 350, 1424, 776, + + 776, 787, 350, 558, 350, 1425, 559, 1429, 350, 350, 350, 350, 350, 350, 468, 468, 468, 468, 468, 468, - 468, 537, 537, 537, 537, 537, 537, 537, 626, 676, - 350, 352, 352, 352, 352, 352, 352, 352, 352, 676, - 1352, 1353, 626, 352, 815, 830, 753, 842, 627, 352, + 468, 540, 540, 540, 540, 540, 540, 540, 638, 688, + 350, 352, 352, 352, 352, 352, 352, 352, 352, 688, + 1430, 1434, 638, 352, 1437, 783, 774, 788, 639, 352, 352, 352, 352, 352, 352, 356, 356, 356, 356, 356, - 356, 356, 627, 830, 753, 842, 356, 753, 626, 831, - 815, 832, 356, 356, 356, 356, 356, 356, 357, 357, - 357, 357, 357, 357, 357, 357, 1358, 831, 627, 832, - 357, 846, 831, 902, 832, 1361, 357, 357, 357, 357, + 356, 356, 639, 783, 774, 788, 356, 774, 638, 784, + 783, 790, 356, 356, 356, 356, 356, 356, 357, 357, + 357, 357, 357, 357, 357, 357, 1438, 784, 639, 790, + 357, 808, 784, 811, 817, 1443, 357, 357, 357, 357, 357, 357, 358, 358, 358, 358, 358, 358, 358, 358, - 360, 360, 360, 360, 360, 360, 360, 846, 871, 902, - 1362, 360, 1370, 833, 835, 837, 871, 360, 360, 360, + 360, 360, 360, 360, 360, 360, 360, 808, 917, 811, + 817, 360, 1446, 791, 789, 795, 917, 360, 360, 360, 360, 360, 360, 364, 364, 364, 364, 364, 364, 364, - 364, 833, 835, 837, 364, 1386, 833, 835, 837, 838, + 364, 791, 789, 795, 364, 789, 824, 797, 797, 1447, 364, 364, 364, 364, 364, 364, 366, 366, 366, 366, - 366, 366, 366, 366, 366, 882, 1387, 838, 366, 914, - 1412, 840, 838, 882, 366, 366, 366, 366, 366, 366, - 367, 367, 367, 367, 367, 367, 367, 367, 894, 840, - 937, 367, 840, 1415, 924, 914, 894, 367, 367, 367, + 366, 366, 366, 366, 366, 797, 836, 839, 366, 851, + 866, 867, 824, 1452, 366, 366, 366, 366, 366, 366, + 367, 367, 367, 367, 367, 367, 367, 367, 866, 867, + 892, 367, 836, 839, 867, 851, 868, 367, 367, 367, 367, 367, 367, 373, 373, 373, 373, 373, 373, 373, - 373, 373, 924, 1426, 1430, 373, 937, 924, 926, 921, - 927, 373, 373, 373, 373, 373, 373, 375, 375, 375, - 375, 375, 375, 375, 375, 375, 926, 921, 927, 375, - 921, 375, 1434, 940, 926, 375, 375, 375, 375, 375, - 375, 542, 542, 542, 542, 542, 542, 542, 546, 546, - 546, 546, 546, 546, 546, 1438, 1442, 375, 396, 940, - 943, 1450, 396, 946, 928, 396, 929, 634, 396, 635, + 373, 373, 1455, 1456, 868, 373, 892, 1461, 879, 868, + 869, 373, 373, 373, 373, 373, 373, 375, 375, 375, + 375, 375, 375, 375, 375, 375, 879, 879, 869, 375, + 692, 375, 1464, 869, 693, 375, 375, 375, 375, 375, + 375, 490, 646, 490, 692, 696, 490, 490, 693, 647, + 490, 490, 699, 928, 700, 490, 646, 375, 396, 696, + 1465, 928, 396, 647, 700, 396, 699, 650, 396, 651, 396, 396, 396, 396, 430, 430, 430, 430, 430, 430, - 430, 634, 928, 635, 929, 430, 943, 933, 1020, 946, + 430, 650, 646, 651, 1473, 430, 780, 871, 873, 647, - 675, 430, 430, 430, 430, 430, 430, 432, 432, 432, - 432, 432, 432, 432, 675, 933, 1020, 634, 432, 635, - 952, 959, 971, 1459, 432, 432, 432, 432, 432, 432, + 1502, 430, 430, 430, 430, 430, 430, 432, 432, 432, + 432, 432, 432, 432, 780, 871, 873, 650, 432, 651, + 871, 873, 948, 780, 432, 432, 432, 432, 432, 432, 433, 433, 433, 433, 433, 433, 433, 433, 435, 435, - 435, 435, 435, 435, 435, 1010, 952, 959, 971, 435, - 974, 986, 990, 1010, 1231, 435, 435, 435, 435, 435, - 435, 438, 438, 438, 438, 438, 438, 438, 438, 1218, - 1214, 638, 639, 438, 1207, 438, 974, 986, 990, 438, - 438, 438, 438, 438, 438, 638, 639, 646, 646, 646, - 646, 646, 646, 646, 648, 648, 648, 648, 648, 648, - - 648, 438, 440, 440, 440, 440, 440, 440, 440, 1002, - 1058, 638, 639, 440, 1028, 1023, 1026, 1206, 1058, 440, + 435, 435, 435, 435, 435, 1503, 1528, 1531, 948, 435, + 1555, 781, 880, 874, 888, 435, 435, 435, 435, 435, + 435, 438, 438, 438, 438, 438, 438, 438, 438, 781, + 880, 874, 888, 438, 1559, 438, 874, 880, 781, 438, + 438, 438, 438, 438, 438, 545, 545, 545, 545, 545, + 545, 545, 549, 549, 549, 549, 549, 549, 549, 767, + + 752, 438, 440, 440, 440, 440, 440, 440, 440, 960, + 940, 993, 996, 440, 752, 883, 1563, 767, 940, 440, 440, 440, 440, 440, 440, 441, 441, 441, 441, 441, - 441, 441, 1028, 1023, 1026, 1002, 441, 1023, 1026, 1048, - 1101, 1113, 441, 441, 441, 441, 441, 441, 442, 442, + 441, 441, 1567, 883, 767, 960, 441, 993, 996, 999, + 752, 883, 441, 441, 441, 441, 441, 441, 442, 442, 442, 442, 442, 442, 442, 442, 445, 445, 445, 445, - 445, 445, 445, 445, 445, 1048, 1101, 1113, 445, 1205, - 445, 1201, 1200, 1197, 445, 445, 445, 445, 445, 445, - 652, 652, 652, 652, 652, 652, 652, 700, 700, 700, - 700, 700, 700, 700, 1120, 1124, 445, 451, 451, 451, - - 451, 451, 451, 451, 451, 451, 1069, 1193, 1192, 451, - 1191, 451, 1120, 1124, 1069, 451, 451, 451, 451, 451, - 451, 705, 705, 705, 705, 705, 705, 705, 709, 709, - 709, 709, 709, 709, 709, 728, 745, 451, 457, 457, - 457, 457, 457, 457, 457, 457, 457, 1187, 1186, 728, - 457, 1130, 1183, 1133, 745, 1027, 457, 457, 457, 457, + 445, 445, 445, 445, 445, 999, 1571, 1591, 445, 1002, + 445, 1600, 1275, 1271, 445, 445, 445, 445, 445, 445, + 658, 658, 658, 658, 658, 658, 658, 660, 660, 660, + 660, 660, 660, 660, 973, 1002, 445, 451, 451, 451, + + 451, 451, 451, 451, 451, 451, 1066, 1270, 1269, 451, + 1265, 451, 973, 973, 1066, 451, 451, 451, 451, 451, + 451, 664, 664, 664, 664, 664, 664, 664, 724, 724, + 724, 724, 724, 724, 724, 753, 768, 451, 457, 457, + 457, 457, 457, 457, 457, 457, 457, 1264, 1261, 753, + 457, 1008, 1257, 1015, 768, 886, 457, 457, 457, 457, 457, 457, 459, 459, 459, 459, 459, 459, 459, 459, - 459, 745, 1179, 1027, 459, 728, 459, 1130, 1027, 1133, - 459, 459, 459, 459, 459, 459, 714, 714, 714, 714, - 714, 714, 714, 716, 716, 716, 716, 716, 716, 716, - - 729, 747, 459, 469, 469, 469, 469, 469, 469, 469, - 469, 469, 1178, 1136, 729, 469, 1139, 1177, 1173, 747, - 732, 469, 469, 469, 469, 469, 469, 470, 470, 470, - 470, 470, 470, 470, 732, 1081, 747, 1172, 470, 1136, - 729, 1169, 1139, 1081, 470, 470, 470, 470, 470, 470, - 518, 733, 518, 518, 518, 518, 518, 518, 518, 1093, - 732, 1166, 1142, 518, 1167, 733, 1119, 1093, 1165, 518, - 518, 518, 518, 518, 518, 519, 519, 519, 519, 519, - 519, 519, 519, 1164, 1119, 1160, 1119, 519, 1142, 519, - 1167, 733, 1159, 519, 519, 519, 519, 519, 519, 720, - - 720, 720, 720, 720, 720, 720, 726, 726, 726, 726, - 726, 726, 726, 783, 836, 519, 521, 521, 521, 521, - 521, 521, 521, 521, 1181, 1156, 1155, 783, 521, 1195, - 1220, 1229, 836, 1123, 521, 521, 521, 521, 521, 521, - 525, 525, 525, 525, 525, 525, 525, 525, 1220, 836, - 1181, 1123, 525, 783, 1154, 1195, 1123, 1229, 525, 525, - 525, 525, 525, 525, 526, 526, 526, 526, 526, 526, - 526, 526, 527, 527, 527, 527, 527, 527, 527, 1213, - 1257, 1150, 1148, 527, 1292, 1329, 1338, 1213, 1257, 527, - 527, 527, 527, 527, 527, 528, 528, 528, 528, 528, - - 528, 528, 528, 530, 530, 530, 530, 530, 530, 530, - 1292, 1329, 1338, 1347, 530, 1356, 1410, 1116, 1117, 1221, - 530, 530, 530, 530, 530, 530, 531, 531, 531, 531, - 531, 531, 531, 531, 531, 1116, 1117, 1221, 531, 1347, - 1118, 1356, 1410, 784, 531, 531, 531, 531, 531, 531, - 532, 532, 532, 532, 532, 532, 532, 784, 1118, 1116, - 1117, 532, 1147, 1222, 1144, 1143, 1140, 532, 532, 532, - 532, 532, 532, 536, 536, 536, 536, 536, 536, 536, - 536, 1222, 1118, 784, 536, 1137, 1134, 1223, 1224, 1121, - 536, 536, 536, 536, 536, 536, 538, 538, 538, 538, - - 538, 538, 538, 538, 538, 1223, 1224, 1121, 538, 1131, - 1127, 1224, 1306, 1115, 538, 538, 538, 538, 538, 538, - 539, 539, 539, 539, 539, 539, 539, 539, 1121, 1114, - 1306, 539, 1111, 1306, 1107, 1307, 1375, 539, 539, 539, - 539, 539, 539, 547, 547, 547, 547, 547, 547, 547, - 547, 547, 1264, 1307, 1375, 547, 1307, 1106, 1308, 1418, - 1264, 547, 547, 547, 547, 547, 547, 548, 548, 548, - 548, 548, 548, 548, 548, 1272, 1308, 1418, 548, 1424, - 1103, 1420, 1098, 1272, 548, 548, 548, 548, 548, 548, - 557, 557, 557, 557, 557, 557, 557, 557, 557, 1420, - - 1308, 1428, 557, 1432, 1436, 1424, 834, 756, 557, 557, - 557, 557, 557, 557, 559, 559, 559, 559, 559, 559, - 559, 559, 559, 746, 834, 756, 559, 1428, 559, 1432, - 1436, 841, 559, 559, 559, 559, 559, 559, 756, 834, - 1097, 746, 763, 763, 763, 763, 763, 763, 763, 841, - 791, 792, 839, 746, 559, 601, 601, 601, 601, 601, - 601, 601, 1440, 1280, 791, 792, 601, 1448, 841, 1447, - 839, 1280, 601, 601, 601, 601, 601, 601, 603, 603, - 603, 603, 603, 603, 603, 839, 1288, 1447, 1440, 603, - 791, 792, 1457, 1448, 1288, 603, 603, 603, 603, 603, - - 603, 604, 604, 604, 604, 604, 604, 604, 604, 606, - 606, 606, 606, 606, 606, 606, 1320, 1369, 1457, 1094, - 606, 1086, 1225, 1085, 1320, 1369, 606, 606, 606, 606, - 606, 606, 608, 608, 608, 608, 608, 608, 608, 608, - 1225, 1082, 1074, 1073, 608, 1070, 1063, 1062, 1059, 1225, - 608, 608, 608, 608, 608, 608, 609, 609, 609, 609, - 609, 609, 609, 609, 610, 610, 610, 610, 610, 610, - 610, 1416, 1054, 1053, 1050, 610, 1049, 1046, 1045, 1416, - 1042, 610, 610, 610, 610, 610, 610, 611, 611, 611, - 611, 611, 611, 611, 611, 614, 614, 614, 614, 614, - - 614, 614, 614, 614, 751, 795, 796, 614, 1039, 614, - 1036, 1032, 1015, 614, 614, 614, 614, 614, 614, 795, - 796, 1011, 751, 804, 804, 804, 804, 804, 804, 804, - 1004, 757, 1003, 806, 751, 614, 617, 617, 617, 617, - 617, 617, 617, 617, 617, 795, 796, 806, 617, 757, - 617, 922, 807, 810, 617, 617, 617, 617, 617, 617, - 1000, 757, 996, 995, 992, 988, 807, 810, 987, 922, - 922, 984, 811, 806, 905, 980, 617, 623, 623, 623, - 623, 623, 623, 623, 623, 623, 811, 922, 905, 623, - 979, 623, 807, 810, 906, 623, 623, 623, 623, 623, - - 623, 819, 819, 819, 819, 819, 819, 819, 906, 976, - 973, 972, 811, 909, 905, 910, 969, 623, 628, 628, - 628, 628, 628, 628, 628, 628, 628, 909, 965, 910, - 628, 964, 961, 1374, 906, 960, 628, 628, 628, 628, - 628, 628, 630, 630, 630, 630, 630, 630, 630, 630, - 630, 1374, 957, 909, 630, 910, 630, 953, 949, 1374, - 630, 630, 630, 630, 630, 630, 821, 821, 821, 821, - 821, 821, 821, 825, 825, 825, 825, 825, 825, 825, - 954, 920, 630, 640, 640, 640, 640, 640, 640, 640, - 640, 640, 948, 947, 954, 640, 944, 941, 938, 920, - - 935, 640, 640, 640, 640, 640, 640, 642, 642, 642, - 642, 642, 642, 642, 642, 642, 920, 934, 916, 642, - 954, 642, 915, 912, 908, 642, 642, 642, 642, 642, - 642, 864, 864, 864, 864, 864, 864, 864, 869, 869, - 869, 869, 869, 869, 869, 925, 917, 642, 653, 653, - 653, 653, 653, 653, 653, 653, 653, 907, 904, 899, - 653, 895, 887, 925, 917, 883, 653, 653, 653, 653, - 653, 653, 654, 654, 654, 654, 654, 654, 654, 917, - 925, 876, 872, 654, 867, 862, 861, 860, 955, 654, - 654, 654, 654, 654, 654, 657, 657, 657, 657, 657, - - 657, 657, 955, 857, 854, 850, 657, 918, 919, 923, - 828, 1022, 657, 657, 657, 657, 657, 657, 873, 873, - 873, 873, 873, 873, 873, 918, 919, 923, 955, 1022, - 657, 680, 824, 680, 680, 680, 680, 680, 680, 680, - 918, 919, 919, 923, 680, 817, 816, 1021, 1022, 1125, - 680, 680, 680, 680, 680, 680, 878, 878, 878, 878, - 878, 878, 878, 962, 963, 1021, 1305, 1125, 680, 681, - 681, 681, 681, 681, 681, 681, 681, 962, 963, 1021, - 813, 681, 1125, 809, 1305, 808, 805, 681, 681, 681, - 681, 681, 681, 880, 880, 880, 880, 880, 880, 880, - - 802, 1305, 801, 962, 963, 681, 682, 682, 682, 682, - 682, 682, 682, 884, 884, 884, 884, 884, 884, 884, - 890, 890, 890, 890, 890, 890, 890, 892, 892, 892, - 892, 892, 892, 892, 896, 896, 896, 896, 896, 896, - 896, 682, 683, 683, 683, 683, 683, 683, 683, 798, - 794, 793, 790, 683, 789, 786, 782, 779, 778, 683, - 683, 683, 683, 683, 683, 685, 685, 685, 685, 685, - 685, 685, 685, 687, 687, 687, 687, 687, 687, 687, - 687, 777, 776, 773, 770, 687, 767, 766, 739, 738, - 735, 687, 687, 687, 687, 687, 687, 688, 688, 688, - - 688, 688, 688, 688, 688, 690, 690, 690, 690, 690, - 690, 690, 731, 730, 727, 723, 690, 719, 712, 708, - 703, 698, 690, 690, 690, 690, 690, 690, 691, 691, - 691, 691, 691, 691, 691, 691, 691, 697, 696, 693, - 691, 689, 679, 678, 674, 966, 691, 691, 691, 691, - 691, 691, 692, 692, 692, 692, 692, 692, 692, 966, - 673, 671, 670, 692, 665, 655, 651, 645, 644, 692, - 692, 692, 692, 692, 692, 694, 694, 694, 694, 694, - 694, 694, 694, 694, 641, 966, 637, 694, 636, 633, - 632, 629, 967, 694, 694, 694, 694, 694, 694, 695, - - 695, 695, 695, 695, 695, 695, 967, 625, 622, 621, - 695, 620, 619, 616, 613, 612, 695, 695, 695, 695, - 695, 695, 699, 699, 699, 699, 699, 699, 699, 699, - 605, 602, 967, 699, 600, 596, 590, 589, 588, 699, - 699, 699, 699, 699, 699, 701, 701, 701, 701, 701, - 701, 701, 701, 701, 587, 583, 582, 701, 580, 579, - 578, 577, 576, 701, 701, 701, 701, 701, 701, 702, - 702, 702, 702, 702, 702, 702, 702, 575, 574, 573, - 702, 572, 571, 570, 569, 566, 702, 702, 702, 702, - 702, 702, 710, 710, 710, 710, 710, 710, 710, 710, - - 710, 565, 564, 562, 710, 561, 558, 553, 549, 545, - 710, 710, 710, 710, 710, 710, 711, 711, 711, 711, - 711, 711, 711, 711, 540, 535, 534, 711, 533, 529, - 523, 522, 517, 711, 711, 711, 711, 711, 711, 721, - 721, 721, 721, 721, 721, 721, 721, 721, 516, 515, - 514, 721, 513, 512, 511, 510, 509, 721, 721, 721, - 721, 721, 721, 722, 722, 722, 722, 722, 722, 722, - 722, 508, 507, 506, 722, 505, 503, 500, 499, 498, - 722, 722, 722, 722, 722, 722, 734, 734, 734, 734, - 734, 734, 734, 734, 734, 497, 496, 495, 734, 494, - - 492, 491, 490, 489, 734, 734, 734, 734, 734, 734, - 736, 736, 736, 736, 736, 736, 736, 736, 736, 487, - 485, 484, 736, 483, 736, 482, 481, 479, 736, 736, - 736, 736, 736, 736, 903, 903, 903, 903, 903, 903, - 903, 975, 975, 975, 975, 975, 975, 975, 977, 978, - 736, 758, 981, 758, 758, 758, 758, 758, 758, 758, - 478, 476, 977, 978, 758, 475, 981, 474, 1025, 758, - 758, 758, 758, 758, 758, 758, 760, 760, 760, 760, - 760, 760, 760, 471, 467, 463, 1025, 760, 977, 978, - 462, 461, 981, 760, 760, 760, 760, 760, 760, 762, - - 762, 762, 762, 762, 762, 762, 1025, 458, 454, 453, - 762, 450, 1373, 449, 448, 447, 762, 762, 762, 762, - 762, 762, 764, 764, 764, 764, 764, 764, 764, 764, - 1373, 444, 443, 434, 764, 431, 429, 428, 427, 1373, - 764, 764, 764, 764, 764, 764, 765, 765, 765, 765, - 765, 765, 765, 765, 768, 768, 768, 768, 768, 768, - 768, 768, 768, 426, 982, 425, 768, 424, 768, 423, - 422, 421, 768, 768, 768, 768, 768, 768, 982, 991, - 991, 991, 991, 991, 991, 991, 420, 419, 418, 417, - 1017, 993, 994, 416, 768, 771, 771, 771, 771, 771, - - 771, 771, 771, 771, 982, 993, 994, 771, 1017, 771, - 1372, 997, 998, 771, 771, 771, 771, 771, 771, 1017, - 414, 412, 411, 409, 408, 997, 998, 405, 1372, 404, - 402, 993, 994, 400, 399, 771, 774, 774, 774, 774, - 774, 774, 774, 774, 774, 398, 393, 1372, 774, 392, - 774, 997, 998, 391, 774, 774, 774, 774, 774, 774, - 1006, 1006, 1006, 1006, 1006, 1006, 1006, 1008, 1008, 1008, - 1008, 1008, 1008, 1008, 389, 388, 774, 780, 780, 780, - 780, 780, 780, 780, 780, 780, 387, 386, 384, 780, - 379, 780, 378, 377, 374, 780, 780, 780, 780, 780, - - 780, 1012, 1012, 1012, 1012, 1012, 1012, 1012, 1051, 1051, - 1051, 1051, 1051, 1051, 1051, 1104, 370, 780, 785, 785, - 785, 785, 785, 785, 785, 785, 785, 368, 363, 1104, - 785, 362, 359, 354, 353, 1024, 785, 785, 785, 785, - 785, 785, 787, 787, 787, 787, 787, 787, 787, 787, - 787, 1018, 1019, 1024, 787, 1104, 787, 345, 343, 342, - 787, 787, 787, 787, 787, 787, 337, 1024, 335, 1018, - 1019, 1056, 1056, 1056, 1056, 1056, 1056, 1056, 1105, 1419, - 1018, 1019, 787, 797, 797, 797, 797, 797, 797, 797, - 797, 797, 1105, 333, 332, 797, 331, 1419, 330, 329, - - 328, 797, 797, 797, 797, 797, 797, 799, 799, 799, - 799, 799, 799, 799, 799, 799, 1419, 327, 1105, 799, - 325, 799, 324, 323, 318, 799, 799, 799, 799, 799, - 799, 1060, 1060, 1060, 1060, 1060, 1060, 1060, 1065, 1065, - 1065, 1065, 1065, 1065, 1065, 1108, 315, 799, 812, 812, - 812, 812, 812, 812, 812, 812, 812, 314, 313, 1108, - 812, 310, 303, 302, 300, 299, 812, 812, 812, 812, - 812, 812, 814, 814, 814, 814, 814, 814, 814, 814, - 814, 298, 296, 294, 814, 1108, 814, 293, 292, 290, - 814, 814, 814, 814, 814, 814, 1067, 1067, 1067, 1067, - - 1067, 1067, 1067, 1071, 1071, 1071, 1071, 1071, 1071, 1071, - 1109, 286, 814, 826, 826, 826, 826, 826, 826, 826, - 826, 826, 285, 283, 1109, 826, 280, 276, 275, 272, - 1151, 826, 826, 826, 826, 826, 826, 827, 827, 827, - 827, 827, 827, 827, 1151, 271, 269, 264, 827, 261, - 1109, 259, 258, 256, 827, 827, 827, 827, 827, 827, - 843, 1152, 843, 843, 843, 843, 843, 843, 843, 253, - 1151, 249, 245, 843, 243, 1152, 241, 240, 843, 843, - 843, 843, 843, 843, 843, 844, 844, 844, 844, 844, - 844, 844, 238, 237, 236, 235, 844, 234, 233, 232, - - 230, 1152, 844, 844, 844, 844, 844, 844, 845, 845, - 845, 845, 845, 845, 845, 845, 228, 226, 225, 223, - 845, 221, 845, 220, 218, 215, 845, 845, 845, 845, - 845, 845, 1077, 1077, 1077, 1077, 1077, 1077, 1077, 1079, - 1079, 1079, 1079, 1079, 1079, 1079, 214, 1157, 845, 847, - 847, 847, 847, 847, 847, 847, 213, 211, 209, 208, - 847, 1157, 205, 204, 202, 201, 847, 847, 847, 847, - 847, 847, 848, 848, 848, 848, 848, 848, 848, 848, - 200, 199, 198, 197, 848, 196, 195, 1157, 194, 193, - 848, 848, 848, 848, 848, 848, 849, 849, 849, 849, - - 849, 849, 849, 849, 851, 851, 851, 851, 851, 851, - 851, 192, 191, 190, 189, 851, 188, 187, 186, 185, - 183, 851, 851, 851, 851, 851, 851, 852, 852, 852, - 852, 852, 852, 852, 852, 852, 182, 179, 178, 852, - 177, 176, 175, 174, 1158, 852, 852, 852, 852, 852, - 852, 853, 853, 853, 853, 853, 853, 853, 1158, 173, - 172, 170, 853, 167, 166, 165, 164, 161, 853, 853, - 853, 853, 853, 853, 855, 855, 855, 855, 855, 855, - 855, 855, 855, 159, 1158, 150, 855, 149, 148, 147, - 146, 1161, 855, 855, 855, 855, 855, 855, 856, 856, - - 856, 856, 856, 856, 856, 1161, 145, 144, 142, 856, - 140, 139, 137, 135, 133, 856, 856, 856, 856, 856, - 856, 858, 858, 858, 858, 858, 858, 858, 858, 858, - 132, 1161, 131, 858, 130, 128, 127, 126, 1162, 858, - 858, 858, 858, 858, 858, 859, 859, 859, 859, 859, - 859, 859, 1162, 125, 124, 123, 859, 121, 119, 118, - 117, 116, 859, 859, 859, 859, 859, 859, 863, 863, - 863, 863, 863, 863, 863, 863, 115, 114, 1162, 863, - 113, 112, 111, 109, 108, 863, 863, 863, 863, 863, - 863, 865, 865, 865, 865, 865, 865, 865, 865, 865, - - 101, 100, 99, 865, 97, 96, 93, 92, 91, 865, - 865, 865, 865, 865, 865, 866, 866, 866, 866, 866, - 866, 866, 866, 90, 89, 88, 866, 87, 86, 85, - 84, 82, 866, 866, 866, 866, 866, 866, 874, 874, - 874, 874, 874, 874, 874, 874, 874, 81, 80, 79, - 874, 78, 77, 76, 75, 74, 874, 874, 874, 874, - 874, 874, 875, 875, 875, 875, 875, 875, 875, 875, - 73, 72, 71, 875, 70, 69, 67, 66, 65, 875, - 875, 875, 875, 875, 875, 885, 885, 885, 885, 885, - 885, 885, 885, 885, 51, 43, 42, 885, 40, 18, - - 11, 8, 3, 885, 885, 885, 885, 885, 885, 886, - 886, 886, 886, 886, 886, 886, 886, 0, 0, 0, - 886, 0, 0, 0, 0, 0, 886, 886, 886, 886, - 886, 886, 897, 897, 897, 897, 897, 897, 897, 897, - 897, 0, 0, 0, 897, 0, 0, 0, 0, 0, - 897, 897, 897, 897, 897, 897, 898, 898, 898, 898, - 898, 898, 898, 898, 0, 0, 0, 898, 0, 0, - 0, 0, 0, 898, 898, 898, 898, 898, 898, 911, - 911, 911, 911, 911, 911, 911, 911, 911, 0, 0, - 0, 911, 0, 0, 0, 0, 0, 911, 911, 911, - - 911, 911, 911, 913, 913, 913, 913, 913, 913, 913, - 913, 913, 0, 0, 0, 913, 0, 913, 0, 0, - 0, 913, 913, 913, 913, 913, 913, 1083, 1083, 1083, - 1083, 1083, 1083, 1083, 1089, 1089, 1089, 1089, 1089, 1089, - 1089, 0, 0, 913, 930, 930, 930, 930, 930, 930, - 930, 930, 930, 0, 0, 0, 0, 930, 0, 0, - 0, 0, 930, 930, 930, 930, 930, 930, 930, 931, - 931, 931, 931, 931, 931, 931, 931, 932, 932, 932, - 932, 932, 932, 932, 932, 932, 1091, 1091, 1091, 1091, - 1091, 1091, 1091, 0, 0, 932, 936, 936, 936, 936, - - 936, 936, 936, 936, 936, 0, 0, 0, 936, 0, - 936, 0, 0, 0, 936, 936, 936, 936, 936, 936, - 1095, 1095, 1095, 1095, 1095, 1095, 1095, 1102, 1102, 1102, - 1102, 1102, 1102, 1102, 1122, 0, 936, 939, 939, 939, - 939, 939, 939, 939, 939, 939, 0, 1170, 1171, 939, - 0, 939, 1122, 0, 0, 939, 939, 939, 939, 939, - 939, 1170, 1171, 1122, 1145, 1145, 1145, 1145, 1145, 1145, - 1145, 0, 0, 0, 0, 1174, 0, 939, 942, 942, - 942, 942, 942, 942, 942, 942, 942, 1170, 1171, 1174, - 942, 0, 942, 0, 0, 1175, 942, 942, 942, 942, - - 942, 942, 1168, 1168, 1168, 1168, 1168, 1168, 1168, 1175, - 0, 0, 0, 0, 1184, 1174, 1185, 0, 942, 945, - 945, 945, 945, 945, 945, 945, 945, 945, 1184, 0, - 1185, 945, 0, 945, 0, 1175, 1188, 945, 945, 945, - 945, 945, 945, 1182, 1182, 1182, 1182, 1182, 1182, 1182, - 1188, 0, 0, 0, 1184, 0, 1185, 0, 1189, 945, - 950, 950, 950, 950, 950, 950, 950, 950, 0, 0, - 0, 950, 1189, 0, 0, 0, 1188, 950, 950, 950, - 950, 950, 950, 951, 951, 951, 951, 951, 951, 951, - 951, 951, 0, 0, 0, 951, 0, 951, 1189, 0, - - 1198, 951, 951, 951, 951, 951, 951, 1196, 1196, 1196, - 1196, 1196, 1196, 1196, 1198, 0, 0, 0, 0, 1199, - 0, 1202, 0, 951, 956, 956, 956, 956, 956, 956, - 956, 956, 956, 1199, 0, 1202, 956, 0, 0, 0, - 1198, 0, 956, 956, 956, 956, 956, 956, 958, 958, - 958, 958, 958, 958, 958, 958, 958, 0, 1203, 1199, - 958, 1202, 958, 0, 0, 0, 958, 958, 958, 958, - 958, 958, 1203, 1209, 1209, 1209, 1209, 1209, 1209, 1209, - 1211, 1211, 1211, 1211, 1211, 1211, 1211, 0, 958, 968, - 968, 968, 968, 968, 968, 968, 968, 968, 1203, 0, - - 0, 968, 0, 0, 0, 0, 0, 968, 968, 968, - 968, 968, 968, 970, 970, 970, 970, 970, 970, 970, - 970, 970, 0, 0, 0, 970, 0, 970, 0, 0, - 1247, 970, 970, 970, 970, 970, 970, 1215, 1215, 1215, - 1215, 1215, 1215, 1215, 1247, 0, 0, 0, 0, 0, - 0, 1248, 0, 970, 983, 983, 983, 983, 983, 983, - 983, 983, 983, 0, 0, 1248, 983, 0, 0, 0, - 1247, 1226, 983, 983, 983, 983, 983, 983, 985, 985, - 985, 985, 985, 985, 985, 985, 985, 1227, 0, 1226, - 985, 1248, 985, 0, 0, 0, 985, 985, 985, 985, - - 985, 985, 1226, 0, 0, 1227, 1255, 1255, 1255, 1255, - 1255, 1255, 1255, 0, 1295, 0, 1227, 0, 985, 999, - 999, 999, 999, 999, 999, 999, 999, 999, 1295, 0, - 0, 999, 0, 0, 0, 0, 0, 999, 999, 999, - 999, 999, 999, 1001, 1001, 1001, 1001, 1001, 1001, 1001, - 1001, 1001, 0, 0, 1295, 1001, 0, 1001, 0, 0, - 0, 1001, 1001, 1001, 1001, 1001, 1001, 1260, 1260, 1260, - 1260, 1260, 1260, 1260, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1296, 0, 1001, 1013, 1013, 1013, 1013, 1013, 1013, - 1013, 1013, 1013, 0, 0, 1296, 1013, 0, 0, 0, - - 0, 1299, 1013, 1013, 1013, 1013, 1013, 1013, 1014, 1014, - 1014, 1014, 1014, 1014, 1014, 1299, 0, 0, 0, 1014, - 0, 1296, 0, 0, 0, 1014, 1014, 1014, 1014, 1014, - 1014, 1029, 1029, 1029, 1029, 1029, 1029, 1029, 1029, 1029, - 0, 1299, 0, 0, 1029, 0, 0, 0, 0, 1029, - 1029, 1029, 1029, 1029, 1029, 1029, 1030, 1030, 1030, 1030, - 1030, 1030, 1030, 1030, 1031, 1031, 1031, 1031, 1031, 1031, - 1031, 1031, 1031, 1268, 1268, 1268, 1268, 1268, 1268, 1268, - 0, 1300, 1031, 1033, 1033, 1033, 1033, 1033, 1033, 1033, - 0, 0, 0, 0, 1033, 1300, 0, 0, 0, 0, - - 1033, 1033, 1033, 1033, 1033, 1033, 1034, 1034, 1034, 1034, - 1034, 1034, 1034, 1034, 1034, 0, 0, 0, 1034, 0, - 0, 1300, 0, 1324, 1034, 1034, 1034, 1034, 1034, 1034, - 1035, 1035, 1035, 1035, 1035, 1035, 1035, 1324, 0, 0, - 0, 1035, 0, 0, 0, 0, 0, 1035, 1035, 1035, - 1035, 1035, 1035, 1037, 1037, 1037, 1037, 1037, 1037, 1037, - 1037, 1037, 0, 1324, 0, 1037, 0, 0, 0, 0, - 1325, 1037, 1037, 1037, 1037, 1037, 1037, 1038, 1038, 1038, - 1038, 1038, 1038, 1038, 1325, 0, 0, 0, 1038, 0, - 0, 0, 0, 0, 1038, 1038, 1038, 1038, 1038, 1038, - - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 0, - 1325, 0, 1040, 0, 0, 0, 0, 1332, 1040, 1040, - 1040, 1040, 1040, 1040, 1041, 1041, 1041, 1041, 1041, 1041, - 1041, 1332, 0, 0, 0, 1041, 0, 0, 0, 0, - 0, 1041, 1041, 1041, 1041, 1041, 1041, 1043, 1043, 1043, - 1043, 1043, 1043, 1043, 1043, 1043, 0, 1332, 0, 1043, - 0, 0, 0, 0, 1333, 1043, 1043, 1043, 1043, 1043, - 1043, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1333, 0, - 0, 0, 1044, 0, 0, 0, 0, 0, 1044, 1044, - 1044, 1044, 1044, 1044, 1047, 1047, 1047, 1047, 1047, 1047, - - 1047, 1047, 1047, 0, 1333, 0, 1047, 0, 1047, 0, - 0, 0, 1047, 1047, 1047, 1047, 1047, 1047, 1270, 1270, - 1270, 1270, 1270, 1270, 1270, 1276, 1276, 1276, 1276, 1276, - 1276, 1276, 1341, 0, 1047, 1052, 1052, 1052, 1052, 1052, - 1052, 1052, 1052, 1052, 0, 0, 1341, 1052, 0, 0, - 0, 0, 0, 1052, 1052, 1052, 1052, 1052, 1052, 1061, - 1061, 1061, 1061, 1061, 1061, 1061, 1061, 1061, 0, 0, - 0, 1061, 1341, 0, 0, 0, 0, 1061, 1061, 1061, - 1061, 1061, 1061, 1072, 1072, 1072, 1072, 1072, 1072, 1072, - 1072, 1072, 0, 0, 0, 1072, 0, 0, 0, 0, - - 0, 1072, 1072, 1072, 1072, 1072, 1072, 1084, 1084, 1084, - 1084, 1084, 1084, 1084, 1084, 1084, 0, 0, 0, 1084, - 0, 0, 0, 0, 0, 1084, 1084, 1084, 1084, 1084, - 1084, 1096, 1096, 1096, 1096, 1096, 1096, 1096, 1096, 1096, - 0, 0, 0, 1096, 0, 0, 0, 0, 0, 1096, - 1096, 1096, 1096, 1096, 1096, 1110, 1110, 1110, 1110, 1110, - 1110, 1110, 1110, 1110, 0, 0, 0, 1110, 0, 0, - 0, 0, 0, 1110, 1110, 1110, 1110, 1110, 1110, 1112, - 1112, 1112, 1112, 1112, 1112, 1112, 1112, 1112, 0, 0, - 0, 1112, 0, 1112, 0, 0, 0, 1112, 1112, 1112, - - 1112, 1112, 1112, 1278, 1278, 1278, 1278, 1278, 1278, 1278, - 1284, 1284, 1284, 1284, 1284, 1284, 1284, 0, 0, 1112, - 1126, 1126, 1126, 1126, 1126, 1126, 1126, 1126, 1126, 1286, - 1286, 1286, 1286, 1286, 1286, 1286, 0, 1342, 1126, 1128, - 1128, 1128, 1128, 1128, 1128, 1128, 1128, 0, 0, 0, - 1128, 1342, 0, 0, 0, 0, 1128, 1128, 1128, 1128, - 1128, 1128, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, - 1129, 0, 0, 0, 1129, 0, 1129, 1342, 0, 0, - 1129, 1129, 1129, 1129, 1129, 1129, 1293, 1293, 1293, 1293, - 1293, 1293, 1293, 1318, 1318, 1318, 1318, 1318, 1318, 1318, - - 0, 0, 1129, 1132, 1132, 1132, 1132, 1132, 1132, 1132, - 1132, 1132, 0, 0, 0, 1132, 0, 1132, 0, 0, - 0, 1132, 1132, 1132, 1132, 1132, 1132, 1330, 1330, 1330, - 1330, 1330, 1330, 1330, 1339, 1339, 1339, 1339, 1339, 1339, - 1339, 1350, 0, 1132, 1135, 1135, 1135, 1135, 1135, 1135, - 1135, 1135, 1135, 0, 0, 1350, 1135, 0, 1135, 0, - 0, 1351, 1135, 1135, 1135, 1135, 1135, 1135, 1348, 1348, - 1348, 1348, 1348, 1348, 1348, 1351, 0, 0, 0, 0, - 1359, 1350, 1360, 0, 1135, 1138, 1138, 1138, 1138, 1138, - 1138, 1138, 1138, 1138, 1359, 0, 1360, 1138, 0, 1138, - - 0, 1351, 1384, 1138, 1138, 1138, 1138, 1138, 1138, 1357, - 1357, 1357, 1357, 1357, 1357, 1357, 1384, 0, 0, 0, - 1359, 0, 1360, 0, 0, 1138, 1141, 1141, 1141, 1141, - 1141, 1141, 1141, 1141, 1141, 0, 0, 0, 1141, 0, - 1141, 0, 1384, 0, 1141, 1141, 1141, 1141, 1141, 1141, - 1365, 1365, 1365, 1365, 1365, 1365, 1365, 1367, 1367, 1367, - 1367, 1367, 1367, 1367, 1385, 0, 1141, 1146, 1146, 1146, - 1146, 1146, 1146, 1146, 1146, 1146, 0, 0, 1385, 1146, - 0, 0, 0, 0, 0, 1146, 1146, 1146, 1146, 1146, - 1146, 1153, 1153, 1153, 1153, 1153, 1153, 1153, 1153, 1153, - - 0, 0, 0, 1153, 1385, 0, 0, 0, 0, 1153, - 1153, 1153, 1153, 1153, 1153, 1163, 1163, 1163, 1163, 1163, - 1163, 1163, 1163, 1163, 0, 0, 0, 1163, 0, 0, - 0, 0, 0, 1163, 1163, 1163, 1163, 1163, 1163, 1176, - 1176, 1176, 1176, 1176, 1176, 1176, 1176, 1176, 0, 0, - 0, 1176, 0, 0, 0, 0, 0, 1176, 1176, 1176, - 1176, 1176, 1176, 1190, 1190, 1190, 1190, 1190, 1190, 1190, - 1190, 1190, 0, 0, 0, 1190, 0, 0, 0, 0, - 0, 1190, 1190, 1190, 1190, 1190, 1190, 1204, 1204, 1204, - 1204, 1204, 1204, 1204, 1204, 1204, 0, 0, 0, 1204, - - 0, 0, 0, 0, 0, 1204, 1204, 1204, 1204, 1204, - 1204, 1216, 1216, 1216, 1216, 1216, 1216, 1216, 1216, 1216, - 0, 0, 0, 1216, 0, 0, 0, 0, 0, 1216, - 1216, 1216, 1216, 1216, 1216, 1228, 1228, 1228, 1228, 1228, - 1228, 1228, 1228, 1228, 0, 0, 0, 1228, 0, 1228, - 0, 0, 0, 1228, 1228, 1228, 1228, 1228, 1228, 1390, - 1390, 1390, 1390, 1390, 1390, 1390, 1394, 1394, 1394, 1394, - 1394, 1394, 1394, 1413, 0, 1228, 1232, 1232, 1232, 1232, - 1232, 1232, 1232, 1232, 1232, 0, 0, 1413, 1232, 0, - 0, 0, 0, 0, 1232, 1232, 1232, 1232, 1232, 1232, - - 1235, 1235, 1235, 1235, 1235, 1235, 1235, 1235, 1235, 0, - 0, 0, 1235, 1413, 0, 0, 0, 0, 1235, 1235, - 1235, 1235, 1235, 1235, 1238, 1238, 1238, 1238, 1238, 1238, - 1238, 1238, 1238, 0, 0, 0, 1238, 0, 0, 0, - 0, 0, 1238, 1238, 1238, 1238, 1238, 1238, 1241, 1241, - 1241, 1241, 1241, 1241, 1241, 1241, 1241, 0, 0, 0, - 1241, 0, 0, 0, 0, 0, 1241, 1241, 1241, 1241, - 1241, 1241, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, - 1244, 0, 0, 0, 1244, 0, 0, 0, 0, 0, - 1244, 1244, 1244, 1244, 1244, 1244, 1249, 1249, 1249, 1249, - - 1249, 1249, 1249, 1249, 1249, 0, 0, 0, 1249, 0, - 0, 0, 0, 0, 1249, 1249, 1249, 1249, 1249, 1249, - 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 0, - 0, 0, 1301, 0, 0, 0, 0, 0, 1301, 1301, - 1301, 1301, 1301, 1301, 1309, 1309, 1309, 1309, 1309, 1309, - 1309, 1309, 1309, 0, 0, 0, 1309, 0, 0, 0, - 0, 0, 1309, 1309, 1309, 1309, 1309, 1309, 1398, 1398, - 1398, 1398, 1398, 1398, 1398, 1402, 1402, 1402, 1402, 1402, - 1402, 1402, 1406, 1406, 1406, 1406, 1406, 1406, 1406, 1411, - 1411, 1411, 1411, 1411, 1411, 1411, 1414, 1422, 1422, 1422, - - 1422, 1422, 1422, 1422, 0, 0, 0, 0, 0, 0, - 1414, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1429, 1429, - 1429, 1429, 1429, 1429, 1429, 1433, 1433, 1433, 1433, 1433, - 1433, 1433, 0, 0, 0, 0, 1414, 1437, 1437, 1437, - 1437, 1437, 1437, 1437, 1441, 1441, 1441, 1441, 1441, 1441, - 1441, 1444, 1444, 1444, 1444, 1444, 1444, 1444, 1449, 1449, - 1449, 1449, 1449, 1449, 1449, 1458, 1458, 1458, 1458, 1458, - 1458, 1458, 1470, 0, 0, 0, 0, 1470, 1470, 1470, - 1471, 0, 1471, 1471, 1471, 1471, 1471, 1471, 1471, 1472, - 0, 1472, 1473, 1473, 1473, 1474, 1474, 1474, 1475, 1475, - - 1475, 1476, 1476, 1476, 1477, 1477, 1477, 1478, 1478, 1478, - 1479, 1479, 1479, 1480, 1480, 1480, 1481, 1481, 1481, 1482, - 0, 1482, 1483, 1483, 1483, 1484, 1484, 1484, 1485, 1485, - 1485, 1486, 1486, 1486, 1487, 0, 1487, 1488, 1488, 1488, - 1489, 1489, 0, 0, 1489, 1490, 1490, 1490, 1491, 1491, - 1491, 1492, 1492, 1492, 1493, 1493, 1493, 1494, 1494, 1494, - 1495, 1495, 1495, 1496, 1496, 1496, 1497, 1497, 1497, 1498, - 1498, 1498, 1499, 1499, 0, 0, 1499, 1500, 1500, 1500, - 1501, 1501, 1501, 1502, 0, 1502, 1503, 1503, 1503, 1504, - 1504, 1504, 1505, 0, 1505, 1506, 1506, 1506, 1507, 1507, - - 1507, 1508, 1508, 1508, 1509, 1509, 1509, 1510, 1510, 1510, - 1511, 0, 1511, 1512, 0, 1512, 1513, 1513, 1513, 1514, - 1514, 1514, 1515, 0, 1515, 1516, 1516, 0, 0, 1516, - 1517, 1517, 0, 0, 1517, 1518, 1518, 1518, 1519, 1519, - 1519, 1520, 1520, 0, 1520, 1521, 1521, 1521, 1522, 1522, - 1522, 1523, 1523, 1523, 1524, 1524, 1524, 1525, 1525, 1525, - 1526, 1526, 1526, 1527, 1527, 1527, 1528, 1528, 0, 0, - 1528, 1529, 1529, 1529, 1530, 1530, 1530, 1531, 1531, 0, - 1531, 1532, 1532, 0, 0, 1532, 1533, 1533, 0, 1533, - 1534, 1534, 1535, 1535, 0, 0, 1535, 1536, 1536, 1536, - - 1537, 1537, 1537, 1538, 1538, 0, 1538, 1539, 0, 1539, - 1540, 0, 1540, 1541, 1541, 1541, 1542, 1542, 1542, 1543, - 0, 1543, 1544, 1544, 1544, 1545, 1545, 1545, 1546, 1546, - 1546, 1547, 1547, 1547, 1548, 1548, 1548, 1549, 1549, 1549, - 1550, 0, 1550, 1551, 0, 1551, 1552, 1552, 1552, 1553, - 1553, 1553, 1554, 0, 1554, 1555, 0, 1555, 1556, 0, - 1556, 1557, 1557, 1557, 1558, 1558, 1558, 1559, 0, 1559, - 1560, 0, 0, 1560, 1561, 1561, 0, 1561, 1562, 1562, - 0, 0, 1562, 1563, 1563, 0, 1563, 1564, 1564, 1565, - 1565, 0, 0, 1565, 1566, 1566, 1566, 1567, 1567, 1567, - - 1568, 1568, 0, 1568, 1569, 1569, 1569, 0, 1569, 1569, - 1570, 1570, 1570, 1571, 1571, 1571, 1572, 1572, 1572, 1573, - 1573, 1573, 1574, 1574, 1574, 1575, 1575, 1575, 1576, 1576, - 1576, 1577, 1577, 1577, 1578, 1578, 0, 0, 1578, 1579, - 1579, 1579, 1580, 1580, 1580, 1581, 1581, 0, 1581, 1582, - 1582, 0, 0, 1582, 1583, 1583, 0, 1583, 1584, 1584, - 1585, 1585, 0, 0, 1585, 1586, 1586, 1586, 1587, 1587, - 1587, 1588, 1588, 0, 1588, 1589, 0, 0, 1589, 1590, - 1590, 0, 1590, 1591, 1591, 0, 0, 1591, 1592, 1592, - 0, 1592, 1593, 1593, 1594, 1594, 0, 0, 1594, 1595, - - 1595, 1595, 1596, 1596, 1596, 1597, 1597, 0, 1597, 1598, - 0, 1598, 1599, 0, 1599, 1600, 0, 1600, 1601, 1601, - 1601, 1602, 1602, 1602, 1603, 0, 1603, 1604, 1604, 1604, - 0, 1604, 1604, 1605, 1605, 1605, 1606, 1606, 1606, 1607, - 1607, 1607, 1608, 1608, 1608, 1609, 1609, 1609, 1610, 1610, - 1610, 1611, 1611, 1611, 1612, 0, 1612, 1613, 0, 1613, - 1614, 1614, 1614, 1615, 1615, 1615, 1616, 0, 1616, 1617, - 0, 1617, 1618, 0, 1618, 1619, 1619, 1619, 1620, 1620, - 1620, 1621, 0, 1621, 1622, 0, 1622, 1623, 0, 1623, - 1624, 0, 1624, 1625, 1625, 1625, 1626, 1626, 1626, 1627, - - 0, 1627, 1628, 0, 1628, 1629, 0, 0, 1629, 1630, - 1630, 0, 1630, 1631, 1631, 0, 0, 1631, 1632, 1632, - 0, 1632, 1633, 1633, 1634, 1634, 0, 0, 1634, 1635, - 1635, 1635, 1636, 1636, 1636, 1637, 1637, 0, 1637, 1638, - 1638, 1638, 0, 1638, 1638, 1639, 1639, 1639, 1640, 1640, - 1640, 1641, 1641, 1641, 1642, 1642, 1642, 1643, 1643, 1643, - 1644, 1644, 1644, 1645, 1645, 1645, 1646, 1646, 1646, 1647, - 1647, 0, 0, 1647, 1648, 1648, 1648, 1649, 1649, 1649, - 1650, 1650, 0, 1650, 1651, 1651, 0, 0, 1651, 1652, - 1652, 0, 1652, 1653, 1653, 1654, 1654, 0, 0, 1654, - - 1655, 1655, 1655, 1656, 1656, 1656, 1657, 1657, 0, 1657, - 1658, 0, 0, 1658, 1659, 1659, 0, 1659, 1660, 1660, - 0, 0, 1660, 1661, 1661, 0, 1661, 1662, 1662, 1663, - 1663, 0, 0, 1663, 1664, 1664, 1664, 1665, 1665, 1665, - 1666, 1666, 0, 1666, 1667, 0, 1667, 1668, 0, 0, - 1668, 1669, 1669, 0, 1669, 1670, 1670, 0, 0, 1670, - 1671, 1671, 0, 1671, 1672, 1672, 1673, 1673, 0, 0, - 1673, 1674, 1674, 1674, 1675, 1675, 1675, 1676, 1676, 0, - 1676, 1677, 0, 1677, 1678, 0, 1678, 1679, 0, 1679, - 1680, 1680, 1680, 1681, 1681, 1681, 1682, 0, 1682, 1683, - - 1683, 1683, 0, 1683, 1683, 1684, 1684, 1684, 1685, 1685, - 1685, 1686, 1686, 1686, 1687, 1687, 1687, 1688, 1688, 1688, - 1689, 1689, 1689, 1690, 1690, 1690, 1691, 1691, 1691, 1692, - 1692, 1692, 1693, 1693, 1693, 1694, 0, 1694, 1695, 0, - 1695, 1696, 1696, 1696, 1697, 1697, 1697, 1698, 1698, 1698, - 1699, 0, 1699, 1700, 0, 1700, 1701, 0, 1701, 1702, - 1702, 1702, 1703, 1703, 1703, 1704, 1704, 1704, 1705, 0, - 1705, 1706, 0, 1706, 1707, 0, 1707, 1708, 0, 1708, - 1709, 1709, 1709, 1710, 1710, 1710, 1711, 1711, 1711, 1712, - 0, 1712, 1713, 0, 1713, 1714, 0, 1714, 1715, 0, - - 1715, 1716, 1716, 1716, 1717, 1717, 1717, 1718, 1718, 1718, - 1719, 0, 1719, 1720, 0, 1720, 1721, 0, 0, 1721, - 1722, 1722, 0, 1722, 1723, 1723, 0, 0, 1723, 1724, - 1724, 0, 1724, 1725, 1725, 1726, 1726, 0, 0, 1726, - 1727, 1727, 1727, 1728, 1728, 1728, 1729, 1729, 0, 1729, - 1730, 1730, 1730, 0, 1730, 1730, 1731, 1731, 1731, 1732, - 1732, 1732, 1733, 1733, 1733, 1734, 1734, 1734, 1735, 1735, - 1735, 1736, 1736, 1736, 1737, 1737, 1737, 1738, 1738, 1738, - 1739, 0, 1739, 1740, 1740, 1740, 1741, 1741, 0, 0, - 1741, 1742, 1742, 1742, 1743, 1743, 1743, 1744, 1744, 0, - - 1744, 1745, 1745, 0, 0, 1745, 1746, 1746, 0, 1746, - 1747, 1747, 1748, 1748, 0, 0, 1748, 1749, 1749, 1749, - 1750, 1750, 1750, 1751, 1751, 0, 1751, 1752, 0, 0, - 1752, 1753, 1753, 0, 1753, 1754, 1754, 0, 0, 1754, - 1755, 1755, 0, 1755, 1756, 1756, 1757, 1757, 0, 0, - 1757, 1758, 1758, 1758, 1759, 1759, 1759, 1760, 1760, 0, - 1760, 1761, 0, 1761, 1762, 0, 0, 1762, 1763, 1763, - 0, 1763, 1764, 1764, 0, 0, 1764, 1765, 1765, 0, - 1765, 1766, 1766, 1767, 1767, 0, 0, 1767, 1768, 1768, - 1768, 1769, 1769, 1769, 1770, 1770, 0, 1770, 1771, 0, - - 1771, 1772, 0, 0, 1772, 1773, 1773, 0, 1773, 1774, - 1774, 0, 0, 1774, 1775, 1775, 0, 1775, 1776, 1776, - 1777, 1777, 0, 0, 1777, 1778, 1778, 1778, 1779, 1779, - 1779, 1780, 1780, 0, 1780, 1781, 0, 1781, 1782, 0, - 1782, 1783, 0, 1783, 1784, 1784, 1784, 1785, 0, 1785, - 1786, 1786, 1786, 1787, 0, 1787, 1788, 1788, 1788, 0, - 1788, 1788, 1789, 0, 1789, 1790, 1790, 1790, 1791, 0, - 1791, 1792, 1792, 1792, 1793, 0, 1793, 1794, 1794, 1794, - 1795, 0, 1795, 1796, 1796, 1796, 1797, 0, 1797, 1798, - 1798, 1798, 1799, 0, 1799, 1800, 1800, 1800, 1801, 1801, - - 0, 0, 1801, 1802, 1802, 1802, 1803, 1803, 1803, 1804, - 1804, 1804, 1805, 1805, 0, 1805, 1806, 1806, 1806, 1807, - 0, 1807, 1808, 1808, 1808, 1809, 1809, 1809, 1810, 0, - 1810, 1811, 0, 1811, 1812, 1812, 1812, 1813, 1813, 1813, - 1814, 0, 1814, 1815, 0, 1815, 1816, 0, 1816, 1817, - 1817, 1817, 1818, 1818, 1818, 1819, 0, 1819, 1820, 0, - 1820, 1821, 0, 1821, 1822, 1822, 1822, 1823, 1823, 1823, - 1824, 0, 1824, 1825, 0, 1825, 1826, 0, 1826, 1827, - 1827, 1827, 1828, 1828, 1828, 1829, 0, 1829, 1830, 0, - 0, 1830, 1831, 1831, 0, 1831, 1832, 1832, 0, 0, - - 1832, 1833, 1833, 0, 1833, 1834, 1834, 1835, 1835, 0, - 0, 1835, 1836, 1836, 1836, 1837, 0, 1837, 1838, 1838, - 0, 1838, 1839, 1839, 1839, 0, 1839, 1839, 1840, 1840, - 1840, 1841, 1841, 1841, 1842, 0, 1842, 1843, 0, 1843, - 1844, 0, 1844, 1845, 0, 1845, 1846, 0, 1846, 1847, - 0, 1847, 1848, 0, 1848, 1849, 1849, 1849, 1850, 1850, - 1850, 1851, 0, 1851, 1852, 1852, 0, 0, 1852, 1853, - 1853, 0, 1853, 1854, 1854, 1855, 0, 1855, 1856, 0, - 0, 1856, 1857, 1857, 0, 1857, 1858, 1858, 0, 0, - 1858, 1859, 1859, 0, 1859, 1860, 1860, 1861, 0, 1861, - - 1862, 0, 1862, 1863, 0, 0, 1863, 1864, 1864, 0, - 1864, 1865, 1865, 0, 0, 1865, 1866, 1866, 0, 1866, - 1867, 1867, 1868, 0, 1868, 1869, 0, 1869, 1870, 0, - 0, 1870, 1871, 1871, 0, 1871, 1872, 1872, 0, 0, - 1872, 1873, 1873, 0, 1873, 1874, 1874, 1875, 0, 1875, - 1876, 0, 1876, 1877, 0, 0, 1877, 1878, 1878, 0, - 1878, 1879, 1879, 0, 0, 1879, 1880, 1880, 0, 1880, - 1881, 1881, 1882, 0, 1882, 1883, 0, 1883, 1884, 0, - 1884, 1885, 0, 1885, 1886, 1886, 1886, 1887, 0, 1887, - 1888, 1888, 1888, 0, 1888, 1888, 1889, 0, 1889, 1890, - - 0, 1890, 1891, 0, 1891, 1892, 0, 1892, 1893, 0, - 1893, 1894, 0, 1894, 1895, 0, 1895, 1896, 1896, 0, - 0, 1896, 1897, 1897, 0, 1897, 1898, 1898, 1899, 0, - 1899, 1900, 0, 1900, 1901, 0, 1901, 1902, 0, 1902, - 1903, 0, 1903, 1904, 0, 1904, 1905, 0, 1905, 1906, - 0, 1906, 1907, 0, 1907, 1908, 0, 1908, 1909, 0, - 0, 1909, 1910, 1910, 0, 0, 1910, 1911, 0, 1911, - 1912, 0, 1912, 1913, 0, 1913, 1914, 0, 0, 1914, - 1915, 0, 0, 1915, 1916, 0, 0, 1916, 1917, 0, - 0, 1917, 1918, 0, 0, 1918, 1919, 0, 1919, 1920, - - 0, 1920, 1921, 0, 0, 1921, 1922, 0, 1922, 1923, - 0, 1923, 1924, 0, 1924, 1925, 0, 1925, 1926, 0, - 1926, 1927, 0, 0, 1927, 1928, 0, 1928, 1929, 0, - 1929, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, 1469, - 1469, 1469, 1469, 1469, 1469 + 459, 768, 1256, 886, 459, 753, 459, 1008, 886, 1015, + 459, 459, 459, 459, 459, 459, 729, 729, 729, 729, + 729, 729, 729, 733, 733, 733, 733, 733, 733, 733, + + 756, 769, 459, 469, 469, 469, 469, 469, 469, 469, + 469, 469, 1255, 1027, 756, 469, 1251, 975, 1250, 769, + 757, 469, 469, 469, 469, 469, 469, 470, 470, 470, + 470, 470, 470, 470, 757, 975, 769, 1247, 470, 1027, + 756, 785, 975, 876, 470, 470, 470, 470, 470, 470, + 521, 819, 521, 521, 521, 521, 521, 521, 521, 785, + 757, 876, 1244, 521, 876, 819, 972, 967, 785, 521, + 521, 521, 521, 521, 521, 522, 522, 522, 522, 522, + 522, 522, 522, 1243, 972, 967, 1242, 522, 967, 522, + 1238, 819, 972, 522, 522, 522, 522, 522, 522, 738, + + 738, 738, 738, 738, 738, 738, 740, 740, 740, 740, + 740, 740, 740, 771, 777, 522, 524, 524, 524, 524, + 524, 524, 524, 524, 1030, 1124, 1237, 1042, 524, 1046, + 1058, 771, 777, 1124, 524, 524, 524, 524, 524, 524, + 528, 528, 528, 528, 528, 528, 528, 528, 771, 1135, + 1030, 777, 528, 1042, 1114, 1046, 1058, 1135, 528, 528, + 528, 528, 528, 528, 529, 529, 529, 529, 529, 529, + 529, 529, 530, 530, 530, 530, 530, 530, 530, 1147, + 1114, 1234, 1233, 530, 1167, 1179, 1208, 1147, 1232, 530, + 530, 530, 530, 530, 530, 531, 531, 531, 531, 531, + + 531, 531, 531, 533, 533, 533, 533, 533, 533, 533, + 1167, 1179, 1208, 1211, 533, 1228, 1226, 970, 974, 977, + 533, 533, 533, 533, 533, 533, 534, 534, 534, 534, + 534, 534, 534, 534, 534, 970, 974, 977, 534, 1211, + 970, 974, 977, 820, 534, 534, 534, 534, 534, 534, + 535, 535, 535, 535, 535, 535, 535, 820, 1159, 1225, + 1214, 535, 1222, 976, 978, 1221, 1159, 535, 535, 535, + 535, 535, 535, 539, 539, 539, 539, 539, 539, 539, + 539, 976, 978, 820, 539, 978, 1214, 979, 981, 976, + 539, 539, 539, 539, 539, 539, 541, 541, 541, 541, + + 541, 541, 541, 541, 541, 979, 981, 979, 541, 1218, + 983, 981, 984, 1215, 541, 541, 541, 541, 541, 541, + 542, 542, 542, 542, 542, 542, 542, 542, 983, 1212, + 984, 542, 1209, 1205, 985, 989, 1076, 542, 542, 542, + 542, 542, 542, 550, 550, 550, 550, 550, 550, 550, + 550, 550, 985, 989, 1076, 550, 1181, 1079, 1082, 1083, + 1180, 550, 550, 550, 550, 550, 550, 551, 551, 551, + 551, 551, 551, 551, 551, 1079, 1082, 1083, 551, 1079, + 1082, 1084, 1083, 1087, 551, 551, 551, 551, 551, 551, + 560, 560, 560, 560, 560, 560, 560, 560, 560, 1084, + + 1177, 1087, 560, 1217, 1220, 1088, 1087, 1173, 560, 560, + 560, 560, 560, 560, 562, 562, 562, 562, 562, 562, + 562, 562, 562, 1088, 1172, 1088, 562, 1169, 562, 1217, + 1220, 1164, 562, 562, 562, 562, 562, 562, 744, 744, + 744, 744, 744, 744, 744, 750, 750, 750, 750, 750, + 750, 750, 872, 778, 562, 613, 613, 613, 613, 613, + 613, 613, 1245, 1163, 1259, 1273, 613, 1319, 1092, 1160, + 872, 778, 613, 613, 613, 613, 613, 613, 615, 615, + 615, 615, 615, 615, 615, 778, 1092, 872, 1245, 615, + 1259, 1273, 1092, 1319, 1152, 615, 615, 615, 615, 615, + + 615, 616, 616, 616, 616, 616, 616, 616, 616, 618, + 618, 618, 618, 618, 618, 618, 1291, 1151, 1148, 1382, + 618, 1432, 882, 1090, 1291, 1093, 618, 618, 618, 618, + 618, 618, 620, 620, 620, 620, 620, 620, 620, 620, + 882, 1090, 1090, 1093, 620, 1382, 1093, 1432, 1441, 882, + 620, 620, 620, 620, 620, 620, 621, 621, 621, 621, + 621, 621, 621, 621, 622, 622, 622, 622, 622, 622, + 622, 1347, 1354, 1140, 1441, 622, 1139, 1136, 1129, 1347, + 1354, 622, 622, 622, 622, 622, 622, 623, 623, 623, + 623, 623, 623, 623, 623, 626, 626, 626, 626, 626, + + 626, 626, 626, 626, 770, 884, 827, 626, 782, 626, + 786, 1182, 1186, 626, 626, 626, 626, 626, 626, 1362, + 827, 1128, 770, 884, 1125, 1120, 782, 1362, 786, 1182, + 1186, 775, 884, 792, 770, 626, 629, 629, 629, 629, + 629, 629, 629, 629, 629, 782, 827, 786, 629, 775, + 629, 792, 1370, 1182, 629, 629, 629, 629, 629, 629, + 1370, 775, 1119, 1116, 792, 799, 799, 799, 799, 799, + 799, 799, 793, 828, 831, 1450, 629, 635, 635, 635, + 635, 635, 635, 635, 635, 635, 1183, 828, 831, 635, + 793, 635, 832, 842, 843, 635, 635, 635, 635, 635, + + 635, 1450, 793, 1378, 1183, 1459, 832, 842, 843, 1184, + 1115, 1378, 1112, 828, 831, 846, 1111, 635, 640, 640, + 640, 640, 640, 640, 640, 640, 640, 1184, 1183, 846, + 640, 1459, 832, 842, 843, 1189, 640, 640, 640, 640, + 640, 640, 642, 642, 642, 642, 642, 642, 642, 642, + 642, 1184, 1108, 1189, 642, 846, 642, 1105, 1189, 847, + 642, 642, 642, 642, 642, 642, 840, 840, 840, 840, + 840, 840, 840, 847, 855, 855, 855, 855, 855, 855, + 855, 1102, 642, 652, 652, 652, 652, 652, 652, 652, + 652, 652, 1423, 1098, 1526, 652, 1553, 1557, 1185, 847, + + 1423, 652, 652, 652, 652, 652, 652, 654, 654, 654, + 654, 654, 654, 654, 654, 654, 1185, 1071, 1185, 654, + 1526, 654, 1553, 1557, 1067, 654, 654, 654, 654, 654, + 654, 857, 857, 857, 857, 857, 857, 857, 861, 861, + 861, 861, 861, 861, 861, 878, 870, 654, 665, 665, + 665, 665, 665, 665, 665, 665, 665, 1472, 1060, 1059, + 665, 1056, 1561, 878, 870, 1472, 665, 665, 665, 665, + 665, 665, 666, 666, 666, 666, 666, 666, 666, 870, + 878, 1052, 1051, 666, 887, 875, 877, 885, 1561, 666, + 666, 666, 666, 666, 666, 669, 669, 669, 669, 669, + + 669, 669, 887, 875, 877, 885, 669, 966, 1078, 951, + 963, 881, 669, 669, 669, 669, 669, 669, 875, 877, + 1565, 887, 885, 951, 1532, 966, 1078, 952, 963, 881, + 669, 704, 1532, 704, 704, 704, 704, 704, 704, 704, + 881, 952, 966, 963, 704, 1078, 1565, 964, 1048, 951, + 704, 704, 704, 704, 704, 704, 910, 910, 910, 910, + 910, 910, 910, 955, 956, 964, 968, 952, 704, 705, + 705, 705, 705, 705, 705, 705, 705, 955, 956, 1044, + 964, 705, 1190, 1298, 968, 968, 1043, 705, 705, 705, + 705, 705, 705, 915, 915, 915, 915, 915, 915, 915, + + 1190, 1298, 968, 955, 956, 705, 706, 706, 706, 706, + 706, 706, 706, 919, 919, 919, 919, 919, 919, 919, + 924, 924, 924, 924, 924, 924, 924, 926, 926, 926, + 926, 926, 926, 926, 930, 930, 930, 930, 930, 930, + 930, 706, 707, 707, 707, 707, 707, 707, 707, 1040, + 1036, 1035, 1032, 707, 1029, 1569, 1589, 1598, 1028, 707, + 707, 707, 707, 707, 707, 709, 709, 709, 709, 709, + 709, 709, 709, 711, 711, 711, 711, 711, 711, 711, + 711, 1569, 1589, 1598, 1025, 711, 1021, 1020, 1017, 1016, + 1013, 711, 711, 711, 711, 711, 711, 712, 712, 712, + + 712, 712, 712, 712, 712, 714, 714, 714, 714, 714, + 714, 714, 1009, 1005, 1004, 1003, 714, 1000, 997, 1201, + 1202, 1187, 714, 714, 714, 714, 714, 714, 715, 715, + 715, 715, 715, 715, 715, 715, 715, 1201, 1202, 1187, + 715, 1201, 994, 1202, 1299, 1010, 715, 715, 715, 715, + 715, 715, 716, 716, 716, 716, 716, 716, 716, 1010, + 1187, 991, 1299, 716, 990, 962, 1300, 961, 1301, 716, + 716, 716, 716, 716, 716, 718, 718, 718, 718, 718, + 718, 718, 718, 718, 1300, 1010, 1301, 718, 958, 1192, + 1302, 1310, 1011, 718, 718, 718, 718, 718, 718, 719, + + 719, 719, 719, 719, 719, 719, 1011, 1192, 1302, 1310, + 719, 954, 1307, 1302, 1310, 1192, 719, 719, 719, 719, + 719, 719, 723, 723, 723, 723, 723, 723, 723, 723, + 1307, 953, 1011, 723, 950, 945, 941, 1314, 1315, 723, + 723, 723, 723, 723, 723, 725, 725, 725, 725, 725, + 725, 725, 725, 725, 1307, 1314, 1315, 725, 933, 929, + 1396, 1397, 922, 725, 725, 725, 725, 725, 725, 726, + 726, 726, 726, 726, 726, 726, 726, 1315, 1396, 1397, + 726, 1396, 1397, 1398, 1399, 1401, 726, 726, 726, 726, + 726, 726, 734, 734, 734, 734, 734, 734, 734, 734, + + 734, 1398, 1399, 1401, 734, 918, 1399, 913, 1402, 908, + 734, 734, 734, 734, 734, 734, 735, 735, 735, 735, + 735, 735, 735, 735, 907, 1398, 1402, 735, 906, 903, + 1402, 1403, 1404, 735, 735, 735, 735, 735, 735, 745, + 745, 745, 745, 745, 745, 745, 745, 745, 900, 1403, + 1404, 745, 896, 1403, 1406, 1409, 1404, 745, 745, 745, + 745, 745, 745, 746, 746, 746, 746, 746, 746, 746, + 746, 864, 1406, 1409, 746, 860, 1406, 1477, 1409, 1478, + 746, 746, 746, 746, 746, 746, 758, 758, 758, 758, + 758, 758, 758, 758, 758, 1477, 853, 1478, 758, 852, + + 849, 1488, 845, 1477, 758, 758, 758, 758, 758, 758, + 760, 760, 760, 760, 760, 760, 760, 760, 760, 1488, + 1488, 844, 760, 841, 760, 838, 837, 834, 760, 760, + 760, 760, 760, 760, 936, 936, 936, 936, 936, 936, + 936, 938, 938, 938, 938, 938, 938, 938, 1018, 969, + 760, 794, 1019, 794, 794, 794, 794, 794, 794, 794, + 830, 829, 1018, 826, 794, 825, 1019, 969, 1081, 794, + 794, 794, 794, 794, 794, 794, 796, 796, 796, 796, + 796, 796, 796, 969, 822, 818, 1081, 796, 1018, 1534, + 1491, 1536, 1019, 796, 796, 796, 796, 796, 796, 798, + + 798, 798, 798, 798, 798, 798, 1081, 1534, 1491, 1536, + 798, 1491, 1303, 1537, 1544, 1548, 798, 798, 798, 798, + 798, 798, 800, 800, 800, 800, 800, 800, 800, 800, + 1303, 1537, 1544, 1548, 800, 815, 1537, 1544, 814, 1303, + 800, 800, 800, 800, 800, 800, 801, 801, 801, 801, + 801, 801, 801, 801, 804, 804, 804, 804, 804, 804, + 804, 804, 804, 813, 812, 809, 804, 806, 804, 803, + 802, 763, 804, 804, 804, 804, 804, 804, 942, 942, + 942, 942, 942, 942, 942, 949, 949, 949, 949, 949, + 949, 949, 965, 971, 804, 807, 807, 807, 807, 807, + + 807, 807, 807, 807, 1073, 1074, 1022, 807, 762, 807, + 965, 971, 1479, 807, 807, 807, 807, 807, 807, 759, + 1022, 755, 1073, 1074, 754, 965, 965, 982, 971, 980, + 1479, 1077, 751, 1073, 1074, 807, 810, 810, 810, 810, + 810, 810, 810, 810, 810, 982, 1022, 980, 810, 1077, + 810, 1479, 1023, 747, 810, 810, 810, 810, 810, 810, + 980, 1033, 982, 1077, 743, 736, 1023, 1031, 1031, 1031, + 1031, 1031, 1031, 1031, 1034, 1033, 810, 816, 816, 816, + 816, 816, 816, 816, 816, 816, 1576, 1075, 1034, 816, + 732, 816, 1023, 1037, 1038, 816, 816, 816, 816, 816, + + 816, 1033, 727, 722, 1576, 1075, 721, 1037, 1038, 720, + 1541, 717, 713, 1049, 1034, 1050, 1075, 816, 821, 821, + 821, 821, 821, 821, 821, 821, 821, 1049, 1541, 1050, + 821, 703, 702, 1037, 1038, 698, 821, 821, 821, 821, + 821, 821, 823, 823, 823, 823, 823, 823, 823, 823, + 823, 1541, 697, 1049, 823, 1050, 823, 1085, 1196, 1053, + 823, 823, 823, 823, 823, 823, 1047, 1047, 1047, 1047, + 1047, 1047, 1047, 1053, 695, 1085, 1196, 694, 1054, 1410, + 1197, 677, 823, 833, 833, 833, 833, 833, 833, 833, + 833, 833, 1054, 1196, 1085, 833, 667, 1410, 1197, 1053, + + 663, 833, 833, 833, 833, 833, 833, 835, 835, 835, + 835, 835, 835, 835, 835, 835, 1197, 1410, 1054, 835, + 657, 835, 656, 653, 649, 835, 835, 835, 835, 835, + 835, 1062, 1062, 1062, 1062, 1062, 1062, 1062, 1064, 1064, + 1064, 1064, 1064, 1064, 1064, 1170, 1094, 835, 848, 848, + 848, 848, 848, 848, 848, 848, 848, 648, 645, 1170, + 848, 644, 641, 637, 1094, 1094, 848, 848, 848, 848, + 848, 848, 850, 850, 850, 850, 850, 850, 850, 850, + 850, 1094, 1094, 634, 850, 1170, 850, 1587, 1486, 1086, + 850, 850, 850, 850, 850, 850, 1068, 1068, 1068, 1068, + + 1068, 1068, 1068, 633, 632, 1587, 1486, 1086, 1200, 1171, + 1174, 631, 850, 862, 862, 862, 862, 862, 862, 862, + 862, 862, 1086, 1171, 1174, 862, 1200, 1486, 1612, 1080, + 1175, 862, 862, 862, 862, 862, 862, 863, 863, 863, + 863, 863, 863, 863, 1175, 1200, 1612, 1080, 863, 1171, + 1174, 1578, 628, 1579, 863, 863, 863, 863, 863, 863, + 889, 1080, 889, 889, 889, 889, 889, 889, 889, 1578, + 1175, 1579, 1578, 889, 1579, 1620, 1580, 1628, 889, 889, + 889, 889, 889, 889, 889, 890, 890, 890, 890, 890, + 890, 890, 625, 1620, 1580, 1628, 890, 1580, 1411, 1089, + + 1585, 1091, 890, 890, 890, 890, 890, 890, 891, 891, + 891, 891, 891, 891, 891, 891, 1411, 1089, 1585, 1091, + 891, 1585, 891, 624, 617, 1411, 891, 891, 891, 891, + 891, 891, 1089, 1091, 1117, 1117, 1117, 1117, 1117, 1117, + 1117, 1122, 1122, 1122, 1122, 1122, 1122, 1122, 891, 893, + 893, 893, 893, 893, 893, 893, 614, 612, 608, 602, + 893, 601, 1476, 1605, 1607, 1608, 893, 893, 893, 893, + 893, 893, 894, 894, 894, 894, 894, 894, 894, 894, + 1476, 1605, 1607, 1608, 894, 1605, 600, 1607, 1608, 1476, + 894, 894, 894, 894, 894, 894, 895, 895, 895, 895, + + 895, 895, 895, 895, 897, 897, 897, 897, 897, 897, + 897, 599, 595, 594, 592, 897, 591, 590, 1633, 1634, + 1622, 897, 897, 897, 897, 897, 897, 898, 898, 898, + 898, 898, 898, 898, 898, 898, 1633, 1634, 1622, 898, + 589, 585, 1634, 1635, 1229, 898, 898, 898, 898, 898, + 898, 899, 899, 899, 899, 899, 899, 899, 1229, 1622, + 584, 1635, 899, 583, 582, 1636, 1635, 1623, 899, 899, + 899, 899, 899, 899, 901, 901, 901, 901, 901, 901, + 901, 901, 901, 1636, 1229, 1623, 901, 581, 1636, 1638, + 1640, 1230, 901, 901, 901, 901, 901, 901, 902, 902, + + 902, 902, 902, 902, 902, 1230, 1623, 1638, 1640, 902, + 580, 579, 1638, 578, 1624, 902, 902, 902, 902, 902, + 902, 904, 904, 904, 904, 904, 904, 904, 904, 904, + 577, 1230, 1624, 904, 576, 575, 1645, 1647, 1235, 904, + 904, 904, 904, 904, 904, 905, 905, 905, 905, 905, + 905, 905, 1235, 1624, 1645, 1647, 905, 1645, 1649, 1647, + 574, 573, 905, 905, 905, 905, 905, 905, 909, 909, + 909, 909, 909, 909, 909, 909, 1649, 572, 1235, 909, + 569, 1649, 1651, 1652, 1657, 909, 909, 909, 909, 909, + 909, 911, 911, 911, 911, 911, 911, 911, 911, 911, + + 1651, 1652, 1657, 911, 568, 1651, 1653, 567, 565, 911, + 911, 911, 911, 911, 911, 912, 912, 912, 912, 912, + 912, 912, 912, 1657, 1653, 1652, 912, 564, 561, 1660, + 556, 1664, 912, 912, 912, 912, 912, 912, 920, 920, + 920, 920, 920, 920, 920, 920, 920, 1660, 1653, 1664, + 920, 552, 1660, 1668, 1669, 548, 920, 920, 920, 920, + 920, 920, 921, 921, 921, 921, 921, 921, 921, 921, + 543, 1668, 1669, 921, 538, 537, 1668, 1669, 1674, 921, + 921, 921, 921, 921, 921, 931, 931, 931, 931, 931, + 931, 931, 931, 931, 536, 532, 1674, 931, 526, 1675, + + 1677, 1674, 525, 931, 931, 931, 931, 931, 931, 932, + 932, 932, 932, 932, 932, 932, 932, 1675, 1677, 520, + 932, 1675, 519, 1679, 1682, 1683, 932, 932, 932, 932, + 932, 932, 943, 943, 943, 943, 943, 943, 943, 943, + 943, 1679, 1682, 1683, 943, 1682, 1679, 1684, 1689, 518, + 943, 943, 943, 943, 943, 943, 944, 944, 944, 944, + 944, 944, 944, 944, 517, 1684, 1689, 944, 516, 515, + 1694, 1689, 1698, 944, 944, 944, 944, 944, 944, 957, + 957, 957, 957, 957, 957, 957, 957, 957, 1694, 1684, + 1698, 957, 514, 513, 1700, 1698, 1703, 957, 957, 957, + + 957, 957, 957, 959, 959, 959, 959, 959, 959, 959, + 959, 959, 1700, 512, 1703, 959, 511, 959, 510, 509, + 508, 959, 959, 959, 959, 959, 959, 1126, 1126, 1126, + 1126, 1126, 1126, 1126, 1131, 1131, 1131, 1131, 1131, 1131, + 1131, 1710, 1712, 959, 986, 986, 986, 986, 986, 986, + 986, 986, 986, 506, 503, 502, 501, 986, 500, 1710, + 1712, 499, 986, 986, 986, 986, 986, 986, 986, 987, + 987, 987, 987, 987, 987, 987, 987, 988, 988, 988, + 988, 988, 988, 988, 988, 988, 1133, 1133, 1133, 1133, + 1133, 1133, 1133, 1701, 1714, 988, 992, 992, 992, 992, + + 992, 992, 992, 992, 992, 498, 497, 495, 992, 494, + 992, 1701, 1714, 493, 992, 992, 992, 992, 992, 992, + 1137, 1137, 1137, 1137, 1137, 1137, 1137, 1143, 1143, 1143, + 1143, 1143, 1143, 1143, 1706, 1701, 992, 995, 995, 995, + 995, 995, 995, 995, 995, 995, 492, 491, 489, 995, + 487, 995, 1706, 485, 484, 995, 995, 995, 995, 995, + 995, 1145, 1145, 1145, 1145, 1145, 1145, 1145, 1149, 1149, + 1149, 1149, 1149, 1149, 1149, 1719, 1706, 995, 998, 998, + 998, 998, 998, 998, 998, 998, 998, 483, 482, 481, + 998, 479, 998, 1719, 478, 476, 998, 998, 998, 998, + + 998, 998, 1155, 1155, 1155, 1155, 1155, 1155, 1155, 1157, + 1157, 1157, 1157, 1157, 1157, 1157, 1721, 1724, 998, 1001, + 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 475, 474, + 471, 1001, 467, 1001, 1721, 1724, 463, 1001, 1001, 1001, + 1001, 1001, 1001, 1161, 1161, 1161, 1161, 1161, 1161, 1161, + 1168, 1168, 1168, 1168, 1168, 1168, 1168, 462, 461, 1001, + 1006, 1006, 1006, 1006, 1006, 1006, 1006, 1006, 458, 454, + 453, 1006, 450, 449, 448, 1191, 1194, 1006, 1006, 1006, + 1006, 1006, 1006, 1007, 1007, 1007, 1007, 1007, 1007, 1007, + 1007, 1007, 1188, 1191, 1194, 1007, 1543, 1007, 1198, 1236, + + 1308, 1007, 1007, 1007, 1007, 1007, 1007, 1194, 1191, 447, + 1188, 444, 443, 1236, 1543, 434, 1198, 1709, 1308, 431, + 429, 1188, 428, 1007, 1012, 1012, 1012, 1012, 1012, 1012, + 1012, 1012, 1012, 1198, 1543, 1709, 1012, 1308, 1709, 1236, + 1195, 1199, 1012, 1012, 1012, 1012, 1012, 1012, 1014, 1014, + 1014, 1014, 1014, 1014, 1014, 1014, 1014, 1193, 1195, 1199, + 1014, 427, 1014, 426, 425, 1309, 1014, 1014, 1014, 1014, + 1014, 1014, 1195, 1199, 424, 1193, 1223, 1223, 1223, 1223, + 1223, 1223, 1223, 1309, 1239, 1311, 1193, 423, 1014, 1024, + 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1239, 422, + + 1309, 1024, 421, 1311, 420, 1203, 419, 1024, 1024, 1024, + 1024, 1024, 1024, 1026, 1026, 1026, 1026, 1026, 1026, 1026, + 1026, 1026, 1311, 1203, 1239, 1026, 418, 1026, 1240, 417, + 416, 1026, 1026, 1026, 1026, 1026, 1026, 1203, 414, 412, + 411, 409, 1240, 1246, 1246, 1246, 1246, 1246, 1246, 1246, + 408, 1248, 1313, 1026, 1039, 1039, 1039, 1039, 1039, 1039, + 1039, 1039, 1039, 405, 404, 1248, 1039, 402, 1240, 400, + 1313, 399, 1039, 1039, 1039, 1039, 1039, 1039, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1313, 1249, 1252, + 1041, 1248, 1041, 1253, 398, 393, 1041, 1041, 1041, 1041, + + 1041, 1041, 1249, 1252, 392, 391, 389, 1253, 1260, 1260, + 1260, 1260, 1260, 1260, 1260, 388, 1262, 1395, 1041, 1055, + 1055, 1055, 1055, 1055, 1055, 1055, 1055, 1055, 1249, 1252, + 1262, 1055, 387, 1253, 386, 1395, 384, 1055, 1055, 1055, + 1055, 1055, 1055, 1057, 1057, 1057, 1057, 1057, 1057, 1057, + 1057, 1057, 1395, 1263, 1266, 1057, 1262, 1057, 1267, 379, + 378, 1057, 1057, 1057, 1057, 1057, 1057, 1263, 1266, 377, + 374, 370, 1267, 1274, 1274, 1274, 1274, 1274, 1274, 1274, + 368, 1276, 1489, 1057, 1069, 1069, 1069, 1069, 1069, 1069, + 1069, 1069, 1069, 1263, 1266, 1276, 1069, 363, 1267, 362, + + 1489, 1277, 1069, 1069, 1069, 1069, 1069, 1069, 1070, 1070, + 1070, 1070, 1070, 1070, 1070, 1277, 359, 1489, 354, 1070, + 353, 1276, 1707, 345, 1713, 1070, 1070, 1070, 1070, 1070, + 1070, 1095, 1095, 1095, 1095, 1095, 1095, 1095, 1095, 1095, + 1707, 1277, 1713, 343, 1095, 1707, 342, 1713, 337, 1095, + 1095, 1095, 1095, 1095, 1095, 1095, 1096, 1096, 1096, 1096, + 1096, 1096, 1096, 1096, 1097, 1097, 1097, 1097, 1097, 1097, + 1097, 1097, 1097, 335, 1725, 1726, 1280, 1281, 1727, 1306, + 333, 1337, 1097, 1099, 1099, 1099, 1099, 1099, 1099, 1099, + 1280, 1281, 1725, 1726, 1099, 1337, 1727, 1306, 1306, 332, + + 1099, 1099, 1099, 1099, 1099, 1099, 1100, 1100, 1100, 1100, + 1100, 1100, 1100, 1100, 1100, 1306, 1280, 1281, 1100, 331, + 330, 1337, 329, 1338, 1100, 1100, 1100, 1100, 1100, 1100, + 1101, 1101, 1101, 1101, 1101, 1101, 1101, 1338, 328, 327, + 325, 1101, 324, 323, 318, 315, 1687, 1101, 1101, 1101, + 1101, 1101, 1101, 1103, 1103, 1103, 1103, 1103, 1103, 1103, + 1103, 1103, 314, 1338, 1687, 1103, 313, 310, 303, 302, + 1385, 1103, 1103, 1103, 1103, 1103, 1103, 1104, 1104, 1104, + 1104, 1104, 1104, 1104, 1385, 1687, 300, 299, 1104, 298, + 296, 294, 293, 1718, 1104, 1104, 1104, 1104, 1104, 1104, + + 1106, 1106, 1106, 1106, 1106, 1106, 1106, 1106, 1106, 292, + 1385, 1718, 1106, 290, 286, 285, 283, 1386, 1106, 1106, + 1106, 1106, 1106, 1106, 1107, 1107, 1107, 1107, 1107, 1107, + 1107, 1386, 1718, 280, 276, 1107, 275, 272, 271, 269, + 264, 1107, 1107, 1107, 1107, 1107, 1107, 1109, 1109, 1109, + 1109, 1109, 1109, 1109, 1109, 1109, 261, 1386, 259, 1109, + 258, 256, 253, 249, 1389, 1109, 1109, 1109, 1109, 1109, + 1109, 1110, 1110, 1110, 1110, 1110, 1110, 1110, 1389, 245, + 243, 241, 1110, 240, 238, 237, 236, 235, 1110, 1110, + 1110, 1110, 1110, 1110, 1113, 1113, 1113, 1113, 1113, 1113, + + 1113, 1113, 1113, 234, 1389, 233, 1113, 232, 1113, 230, + 228, 226, 1113, 1113, 1113, 1113, 1113, 1113, 1287, 1287, + 1287, 1287, 1287, 1287, 1287, 1289, 1289, 1289, 1289, 1289, + 1289, 1289, 1312, 225, 1113, 1118, 1118, 1118, 1118, 1118, + 1118, 1118, 1118, 1118, 223, 221, 220, 1118, 218, 215, + 1312, 1312, 214, 1118, 1118, 1118, 1118, 1118, 1118, 1127, + 1127, 1127, 1127, 1127, 1127, 1127, 1127, 1127, 1312, 213, + 211, 1127, 209, 208, 205, 204, 202, 1127, 1127, 1127, + 1127, 1127, 1127, 1138, 1138, 1138, 1138, 1138, 1138, 1138, + 1138, 1138, 201, 200, 199, 1138, 198, 197, 196, 195, + + 194, 1138, 1138, 1138, 1138, 1138, 1138, 1150, 1150, 1150, + 1150, 1150, 1150, 1150, 1150, 1150, 193, 192, 191, 1150, + 190, 189, 188, 187, 186, 1150, 1150, 1150, 1150, 1150, + 1150, 1162, 1162, 1162, 1162, 1162, 1162, 1162, 1162, 1162, + 185, 183, 182, 1162, 179, 178, 177, 176, 175, 1162, + 1162, 1162, 1162, 1162, 1162, 1176, 1176, 1176, 1176, 1176, + 1176, 1176, 1176, 1176, 174, 173, 172, 1176, 170, 167, + 166, 165, 164, 1176, 1176, 1176, 1176, 1176, 1176, 1178, + 1178, 1178, 1178, 1178, 1178, 1178, 1178, 1178, 161, 159, + 150, 1178, 149, 1178, 1475, 1535, 1390, 1178, 1178, 1178, + + 1178, 1178, 1178, 1293, 1293, 1293, 1293, 1293, 1293, 1293, + 1390, 1650, 1475, 1535, 148, 1546, 1304, 1316, 147, 1178, + 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1650, + 146, 1475, 1535, 1546, 1304, 1316, 1390, 1427, 1204, 1206, + 1206, 1206, 1206, 1206, 1206, 1206, 1206, 1304, 1316, 1650, + 1206, 1427, 1546, 145, 144, 1400, 1206, 1206, 1206, 1206, + 1206, 1206, 1207, 1207, 1207, 1207, 1207, 1207, 1207, 1207, + 1207, 1305, 1317, 1400, 1207, 142, 1207, 1427, 140, 139, + 1207, 1207, 1207, 1207, 1207, 1207, 137, 1400, 135, 1305, + 1317, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 133, 132, + + 1305, 1317, 1207, 1210, 1210, 1210, 1210, 1210, 1210, 1210, + 1210, 1210, 131, 130, 128, 1210, 127, 1210, 126, 125, + 124, 1210, 1210, 1210, 1210, 1210, 1210, 1350, 1350, 1350, + 1350, 1350, 1350, 1350, 1352, 1352, 1352, 1352, 1352, 1352, + 1352, 123, 121, 1210, 1213, 1213, 1213, 1213, 1213, 1213, + 1213, 1213, 1213, 119, 118, 117, 1213, 116, 1213, 115, + 114, 113, 1213, 1213, 1213, 1213, 1213, 1213, 1358, 1358, + 1358, 1358, 1358, 1358, 1358, 1360, 1360, 1360, 1360, 1360, + 1360, 1360, 112, 111, 1213, 1216, 1216, 1216, 1216, 1216, + 1216, 1216, 1216, 1216, 109, 108, 101, 1216, 100, 1216, + + 99, 97, 96, 1216, 1216, 1216, 1216, 1216, 1216, 1366, + 1366, 1366, 1366, 1366, 1366, 1366, 1368, 1368, 1368, 1368, + 1368, 1368, 1368, 93, 92, 1216, 1219, 1219, 1219, 1219, + 1219, 1219, 1219, 1219, 1219, 91, 90, 89, 1219, 88, + 1219, 87, 86, 85, 1219, 1219, 1219, 1219, 1219, 1219, + 1374, 1374, 1374, 1374, 1374, 1374, 1374, 1376, 1376, 1376, + 1376, 1376, 1376, 1376, 1428, 1539, 1219, 1224, 1224, 1224, + 1224, 1224, 1224, 1224, 1224, 1224, 84, 82, 1428, 1224, + 81, 80, 79, 1539, 78, 1224, 1224, 1224, 1224, 1224, + 1224, 1231, 1231, 1231, 1231, 1231, 1231, 1231, 1231, 1231, + + 77, 1539, 76, 1231, 1428, 75, 74, 73, 72, 1231, + 1231, 1231, 1231, 1231, 1231, 1241, 1241, 1241, 1241, 1241, + 1241, 1241, 1241, 1241, 71, 70, 69, 1241, 67, 66, + 65, 51, 43, 1241, 1241, 1241, 1241, 1241, 1241, 1254, + 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 42, 40, + 18, 1254, 11, 8, 3, 0, 0, 1254, 1254, 1254, + 1254, 1254, 1254, 1268, 1268, 1268, 1268, 1268, 1268, 1268, + 1268, 1268, 0, 0, 0, 1268, 0, 0, 0, 0, + 0, 1268, 1268, 1268, 1268, 1268, 1268, 1282, 1282, 1282, + 1282, 1282, 1282, 1282, 1282, 1282, 0, 0, 0, 1282, + + 0, 0, 0, 0, 0, 1282, 1282, 1282, 1282, 1282, + 1282, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 0, 0, 0, 1294, 0, 0, 0, 0, 0, 1294, + 1294, 1294, 1294, 1294, 1294, 1318, 1318, 1318, 1318, 1318, + 1318, 1318, 1318, 1318, 0, 0, 0, 1318, 0, 1318, + 0, 0, 0, 1318, 1318, 1318, 1318, 1318, 1318, 1383, + 1383, 1383, 1383, 1383, 1383, 1383, 1421, 1421, 1421, 1421, + 1421, 1421, 1421, 1435, 1540, 1318, 1322, 1322, 1322, 1322, + 1322, 1322, 1322, 1322, 1322, 0, 0, 1435, 1322, 0, + 0, 0, 1540, 0, 1322, 1322, 1322, 1322, 1322, 1322, + + 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 0, + 1540, 0, 1325, 1435, 0, 0, 0, 0, 1325, 1325, + 1325, 1325, 1325, 1325, 1328, 1328, 1328, 1328, 1328, 1328, + 1328, 1328, 1328, 0, 0, 0, 1328, 0, 0, 0, + 0, 0, 1328, 1328, 1328, 1328, 1328, 1328, 1331, 1331, + 1331, 1331, 1331, 1331, 1331, 1331, 1331, 0, 0, 0, + 1331, 0, 0, 0, 0, 0, 1331, 1331, 1331, 1331, + 1331, 1331, 1334, 1334, 1334, 1334, 1334, 1334, 1334, 1334, + 1334, 0, 0, 0, 1334, 0, 0, 0, 0, 0, + 1334, 1334, 1334, 1334, 1334, 1334, 1339, 1339, 1339, 1339, + + 1339, 1339, 1339, 1339, 1339, 0, 0, 0, 1339, 0, + 0, 0, 1407, 0, 1339, 1339, 1339, 1339, 1339, 1339, + 1391, 1391, 1391, 1391, 1391, 1391, 1391, 1391, 1391, 1405, + 1407, 1408, 1391, 0, 1436, 0, 0, 0, 1391, 1391, + 1391, 1391, 1391, 1391, 1407, 1490, 0, 1405, 1436, 1408, + 1433, 1433, 1433, 1433, 1433, 1433, 1433, 0, 1405, 0, + 0, 1408, 1444, 1490, 1405, 1412, 1412, 1412, 1412, 1412, + 1412, 1412, 1412, 1412, 1436, 1490, 1444, 1412, 0, 0, + 0, 0, 1445, 1412, 1412, 1412, 1412, 1412, 1412, 1442, + 1442, 1442, 1442, 1442, 1442, 1442, 1445, 1453, 1454, 0, + + 0, 0, 1444, 1451, 1451, 1451, 1451, 1451, 1451, 1451, + 1462, 1453, 1454, 1460, 1460, 1460, 1460, 1460, 1460, 1460, + 1463, 0, 1445, 0, 1462, 1468, 1468, 1468, 1468, 1468, + 1468, 1468, 0, 1480, 1463, 1485, 1481, 1453, 1454, 1470, + 1470, 1470, 1470, 1470, 1470, 1470, 1482, 1483, 1484, 0, + 1462, 1480, 1487, 1485, 1481, 1500, 0, 1501, 1542, 1547, + 1463, 1603, 1480, 1481, 1482, 1483, 1484, 1485, 0, 1500, + 1487, 1501, 0, 1482, 1483, 1484, 1542, 1547, 0, 1603, + 0, 1487, 1506, 1506, 1506, 1506, 1506, 1506, 1506, 0, + 0, 0, 0, 1603, 1542, 1500, 1547, 1501, 1510, 1510, + + 1510, 1510, 1510, 1510, 1510, 1514, 1514, 1514, 1514, 1514, + 1514, 1514, 1518, 1518, 1518, 1518, 1518, 1518, 1518, 1522, + 1522, 1522, 1522, 1522, 1522, 1522, 1527, 1527, 1527, 1527, + 1527, 1527, 1527, 1529, 1530, 1538, 1545, 1549, 1551, 1551, + 1551, 1551, 1551, 1551, 1551, 0, 0, 1529, 1530, 0, + 0, 0, 0, 1538, 1545, 1549, 0, 0, 0, 0, + 0, 0, 1538, 1545, 1549, 1554, 1554, 1554, 1554, 1554, + 1554, 1554, 0, 1529, 1530, 1558, 1558, 1558, 1558, 1558, + 1558, 1558, 1562, 1562, 1562, 1562, 1562, 1562, 1562, 1566, + 1566, 1566, 1566, 1566, 1566, 1566, 1570, 1570, 1570, 1570, + + 1570, 1570, 1570, 1573, 1573, 1573, 1573, 1573, 1573, 1573, + 1577, 1581, 0, 1582, 1583, 1586, 1588, 1584, 1590, 1590, + 1590, 1590, 1590, 1590, 1590, 1601, 1609, 0, 1577, 1581, + 1581, 1582, 1583, 1586, 1588, 1584, 0, 0, 0, 0, + 0, 0, 1586, 1601, 1609, 1577, 1602, 1581, 1604, 1606, + 1582, 1583, 1584, 1588, 1599, 1599, 1599, 1599, 1599, 1599, + 1599, 1610, 1601, 1609, 1602, 1611, 1604, 1606, 1621, 1625, + 0, 1626, 1627, 1602, 1629, 1630, 1637, 1639, 0, 1610, + 1604, 1606, 1641, 1611, 0, 0, 1621, 1625, 1610, 1626, + 1627, 1642, 1629, 1630, 1637, 1639, 1639, 1611, 1625, 0, + + 1641, 1643, 1626, 1637, 1644, 1621, 1630, 1627, 0, 1642, + 1646, 1629, 1648, 1639, 1639, 1654, 1641, 1641, 0, 1643, + 1655, 0, 1644, 1665, 1656, 1658, 1642, 0, 1646, 0, + 1648, 1659, 1661, 1654, 0, 1662, 1643, 1663, 1655, 1644, + 0, 1665, 1656, 1658, 1648, 1646, 1656, 1655, 0, 1659, + 1661, 1666, 1654, 1662, 1658, 1663, 1663, 1667, 1656, 1656, + 1665, 1670, 1671, 1672, 1673, 0, 1659, 1676, 1661, 1666, + 1662, 1678, 1680, 1663, 0, 1667, 1681, 0, 1685, 1670, + 1671, 1672, 1673, 1666, 0, 1676, 1686, 0, 1670, 1678, + 1680, 1690, 1671, 0, 1681, 1667, 1685, 1673, 1672, 1676, + + 1688, 0, 1678, 1691, 1686, 1680, 1692, 1693, 0, 1690, + 1685, 1681, 1696, 1695, 1697, 1699, 0, 0, 1688, 1686, + 1686, 1691, 1702, 0, 1692, 1693, 1690, 1704, 1708, 1688, + 1696, 1695, 1697, 1699, 1693, 1691, 1705, 1711, 1716, 0, + 1702, 1696, 1699, 1692, 1695, 1704, 1708, 1715, 1720, 1717, + 1722, 0, 1697, 0, 1705, 1711, 1716, 1702, 0, 0, + 0, 0, 0, 1708, 1704, 1715, 1720, 1717, 1722, 0, + 0, 0, 1705, 0, 1711, 1716, 1717, 1723, 0, 0, + 1723, 1723, 0, 1715, 0, 1720, 0, 1722, 0, 0, + 0, 0, 0, 0, 0, 1723, 1729, 0, 0, 0, + + 0, 1729, 1729, 1729, 1730, 0, 1730, 1730, 1730, 1730, + 1730, 1730, 1730, 1731, 0, 1731, 1732, 1732, 1732, 1733, + 1733, 1733, 1734, 1734, 1734, 1735, 1735, 1735, 1736, 1736, + 1736, 1737, 1737, 1737, 1738, 1738, 1738, 1739, 1739, 1739, + 1740, 1740, 1740, 1741, 0, 1741, 1742, 1742, 1742, 1743, + 1743, 1743, 1744, 1744, 1744, 1745, 1745, 1745, 1746, 0, + 1746, 1747, 1747, 1747, 1748, 1748, 0, 0, 1748, 1749, + 1749, 1749, 1750, 1750, 1750, 1751, 1751, 1751, 1752, 1752, + 1752, 1753, 1753, 1753, 1754, 1754, 1754, 1755, 1755, 1755, + 1756, 1756, 1756, 1757, 1757, 1757, 1758, 1758, 0, 0, + + 1758, 1759, 1759, 1759, 1760, 1760, 1760, 1761, 0, 1761, + 1762, 1762, 1762, 1763, 1763, 1763, 1764, 0, 1764, 1765, + 1765, 1765, 1766, 1766, 1766, 1767, 1767, 1767, 1768, 1768, + 1768, 1769, 1769, 1769, 1770, 0, 1770, 1771, 0, 1771, + 1772, 1772, 1772, 1773, 1773, 1773, 1774, 0, 1774, 1775, + 1775, 0, 0, 1775, 1776, 1776, 0, 0, 1776, 1777, + 1777, 1777, 1778, 1778, 1778, 1779, 1779, 0, 1779, 1780, + 1780, 1780, 1781, 1781, 1781, 1782, 1782, 1782, 1783, 1783, + 1783, 1784, 1784, 1784, 1785, 1785, 1785, 1786, 1786, 1786, + 1787, 1787, 0, 0, 1787, 1788, 1788, 1788, 1789, 1789, + + 1789, 1790, 1790, 0, 1790, 1791, 1791, 0, 0, 1791, + 1792, 1792, 0, 1792, 1793, 1793, 1794, 1794, 0, 0, + 1794, 1795, 1795, 1795, 1796, 1796, 1796, 1797, 1797, 0, + 1797, 1798, 0, 1798, 1799, 0, 1799, 1800, 1800, 1800, + 1801, 1801, 1801, 1802, 0, 1802, 1803, 1803, 1803, 1804, + 1804, 1804, 1805, 1805, 1805, 1806, 1806, 1806, 1807, 1807, + 1807, 1808, 1808, 1808, 1809, 0, 1809, 1810, 0, 1810, + 1811, 1811, 1811, 1812, 1812, 1812, 1813, 0, 1813, 1814, + 0, 1814, 1815, 0, 1815, 1816, 1816, 1816, 1817, 1817, + 1817, 1818, 0, 1818, 1819, 0, 0, 1819, 1820, 1820, + + 0, 1820, 1821, 1821, 0, 0, 1821, 1822, 1822, 0, + 1822, 1823, 1823, 1824, 1824, 0, 0, 1824, 1825, 1825, + 1825, 1826, 1826, 1826, 1827, 1827, 0, 1827, 1828, 1828, + 1828, 0, 1828, 1828, 1829, 1829, 1829, 1830, 1830, 1830, + 1831, 1831, 1831, 1832, 1832, 1832, 1833, 1833, 1833, 1834, + 1834, 1834, 1835, 1835, 1835, 1836, 1836, 1836, 1837, 1837, + 0, 0, 1837, 1838, 1838, 1838, 1839, 1839, 1839, 1840, + 1840, 0, 1840, 1841, 1841, 0, 0, 1841, 1842, 1842, + 0, 1842, 1843, 1843, 1844, 1844, 0, 0, 1844, 1845, + 1845, 1845, 1846, 1846, 1846, 1847, 1847, 0, 1847, 1848, + + 0, 0, 1848, 1849, 1849, 0, 1849, 1850, 1850, 0, + 0, 1850, 1851, 1851, 0, 1851, 1852, 1852, 1853, 1853, + 0, 0, 1853, 1854, 1854, 1854, 1855, 1855, 1855, 1856, + 1856, 0, 1856, 1857, 0, 1857, 1858, 0, 1858, 1859, + 0, 1859, 1860, 1860, 1860, 1861, 1861, 1861, 1862, 0, + 1862, 1863, 1863, 1863, 0, 1863, 1863, 1864, 1864, 1864, + 1865, 1865, 1865, 1866, 1866, 1866, 1867, 1867, 1867, 1868, + 1868, 1868, 1869, 1869, 1869, 1870, 1870, 1870, 1871, 0, + 1871, 1872, 0, 1872, 1873, 1873, 1873, 1874, 1874, 1874, + 1875, 0, 1875, 1876, 0, 1876, 1877, 0, 1877, 1878, + + 1878, 1878, 1879, 1879, 1879, 1880, 0, 1880, 1881, 0, + 1881, 1882, 0, 1882, 1883, 0, 1883, 1884, 1884, 1884, + 1885, 1885, 1885, 1886, 0, 1886, 1887, 0, 1887, 1888, + 0, 0, 1888, 1889, 1889, 0, 1889, 1890, 1890, 0, + 0, 1890, 1891, 1891, 0, 1891, 1892, 1892, 1893, 1893, + 0, 0, 1893, 1894, 1894, 1894, 1895, 1895, 1895, 1896, + 1896, 0, 1896, 1897, 1897, 1897, 0, 1897, 1897, 1898, + 1898, 1898, 1899, 1899, 1899, 1900, 1900, 1900, 1901, 1901, + 1901, 1902, 1902, 1902, 1903, 1903, 1903, 1904, 1904, 1904, + 1905, 1905, 1905, 1906, 1906, 0, 0, 1906, 1907, 1907, + + 1907, 1908, 1908, 1908, 1909, 1909, 0, 1909, 1910, 1910, + 0, 0, 1910, 1911, 1911, 0, 1911, 1912, 1912, 1913, + 1913, 0, 0, 1913, 1914, 1914, 1914, 1915, 1915, 1915, + 1916, 1916, 0, 1916, 1917, 0, 0, 1917, 1918, 1918, + 0, 1918, 1919, 1919, 0, 0, 1919, 1920, 1920, 0, + 1920, 1921, 1921, 1922, 1922, 0, 0, 1922, 1923, 1923, + 1923, 1924, 1924, 1924, 1925, 1925, 0, 1925, 1926, 0, + 1926, 1927, 0, 0, 1927, 1928, 1928, 0, 1928, 1929, + 1929, 0, 0, 1929, 1930, 1930, 0, 1930, 1931, 1931, + 1932, 1932, 0, 0, 1932, 1933, 1933, 1933, 1934, 1934, + + 1934, 1935, 1935, 0, 1935, 1936, 0, 1936, 1937, 0, + 1937, 1938, 0, 1938, 1939, 1939, 1939, 1940, 1940, 1940, + 1941, 0, 1941, 1942, 1942, 1942, 0, 1942, 1942, 1943, + 1943, 1943, 1944, 1944, 1944, 1945, 1945, 1945, 1946, 1946, + 1946, 1947, 1947, 1947, 1948, 1948, 1948, 1949, 1949, 1949, + 1950, 1950, 1950, 1951, 1951, 1951, 1952, 1952, 1952, 1953, + 0, 1953, 1954, 0, 1954, 1955, 1955, 1955, 1956, 1956, + 1956, 1957, 1957, 1957, 1958, 0, 1958, 1959, 0, 1959, + 1960, 0, 1960, 1961, 1961, 1961, 1962, 1962, 1962, 1963, + 1963, 1963, 1964, 0, 1964, 1965, 0, 1965, 1966, 0, + + 1966, 1967, 0, 1967, 1968, 1968, 1968, 1969, 1969, 1969, + 1970, 1970, 1970, 1971, 0, 1971, 1972, 0, 1972, 1973, + 0, 1973, 1974, 0, 1974, 1975, 1975, 1975, 1976, 1976, + 1976, 1977, 1977, 1977, 1978, 0, 1978, 1979, 0, 1979, + 1980, 0, 0, 1980, 1981, 1981, 0, 1981, 1982, 1982, + 0, 0, 1982, 1983, 1983, 0, 1983, 1984, 1984, 1985, + 1985, 0, 0, 1985, 1986, 1986, 1986, 1987, 1987, 1987, + 1988, 1988, 0, 1988, 1989, 1989, 1989, 0, 1989, 1989, + 1990, 1990, 1990, 1991, 1991, 1991, 1992, 1992, 1992, 1993, + 1993, 1993, 1994, 1994, 1994, 1995, 1995, 1995, 1996, 1996, + + 1996, 1997, 1997, 1997, 1998, 0, 1998, 1999, 1999, 1999, + 2000, 2000, 0, 0, 2000, 2001, 2001, 2001, 2002, 2002, + 2002, 2003, 2003, 0, 2003, 2004, 2004, 0, 0, 2004, + 2005, 2005, 0, 2005, 2006, 2006, 2007, 2007, 0, 0, + 2007, 2008, 2008, 2008, 2009, 2009, 2009, 2010, 2010, 0, + 2010, 2011, 0, 0, 2011, 2012, 2012, 0, 2012, 2013, + 2013, 0, 0, 2013, 2014, 2014, 0, 2014, 2015, 2015, + 2016, 2016, 0, 0, 2016, 2017, 2017, 2017, 2018, 2018, + 2018, 2019, 2019, 0, 2019, 2020, 0, 2020, 2021, 0, + 0, 2021, 2022, 2022, 0, 2022, 2023, 2023, 0, 0, + + 2023, 2024, 2024, 0, 2024, 2025, 2025, 2026, 2026, 0, + 0, 2026, 2027, 2027, 2027, 2028, 2028, 2028, 2029, 2029, + 0, 2029, 2030, 0, 2030, 2031, 0, 0, 2031, 2032, + 2032, 0, 2032, 2033, 2033, 0, 0, 2033, 2034, 2034, + 0, 2034, 2035, 2035, 2036, 2036, 0, 0, 2036, 2037, + 2037, 2037, 2038, 2038, 2038, 2039, 2039, 0, 2039, 2040, + 0, 2040, 2041, 0, 2041, 2042, 0, 2042, 2043, 2043, + 2043, 2044, 0, 2044, 2045, 2045, 2045, 2046, 0, 2046, + 2047, 2047, 2047, 0, 2047, 2047, 2048, 0, 2048, 2049, + 2049, 2049, 2050, 0, 2050, 2051, 2051, 2051, 2052, 0, + + 2052, 2053, 2053, 2053, 2054, 0, 2054, 2055, 2055, 2055, + 2056, 0, 2056, 2057, 2057, 2057, 2058, 0, 2058, 2059, + 2059, 2059, 2060, 2060, 0, 0, 2060, 2061, 2061, 2061, + 2062, 2062, 2062, 2063, 2063, 2063, 2064, 2064, 0, 2064, + 2065, 2065, 2065, 2066, 0, 2066, 2067, 2067, 2067, 2068, + 2068, 2068, 2069, 0, 2069, 2070, 0, 2070, 2071, 2071, + 2071, 2072, 2072, 2072, 2073, 0, 2073, 2074, 0, 2074, + 2075, 0, 2075, 2076, 2076, 2076, 2077, 2077, 2077, 2078, + 0, 2078, 2079, 0, 2079, 2080, 0, 2080, 2081, 2081, + 2081, 2082, 2082, 2082, 2083, 0, 2083, 2084, 0, 2084, + + 2085, 0, 2085, 2086, 2086, 2086, 2087, 2087, 2087, 2088, + 0, 2088, 2089, 0, 0, 2089, 2090, 2090, 0, 2090, + 2091, 2091, 0, 0, 2091, 2092, 2092, 0, 2092, 2093, + 2093, 2094, 2094, 0, 0, 2094, 2095, 2095, 2095, 2096, + 0, 2096, 2097, 2097, 0, 2097, 2098, 2098, 2098, 0, + 2098, 2098, 2099, 2099, 2099, 2100, 2100, 2100, 2101, 0, + 2101, 2102, 0, 2102, 2103, 0, 2103, 2104, 0, 2104, + 2105, 0, 2105, 2106, 0, 2106, 2107, 0, 2107, 2108, + 2108, 2108, 2109, 2109, 2109, 2110, 0, 2110, 2111, 2111, + 0, 0, 2111, 2112, 2112, 0, 2112, 2113, 2113, 2114, + + 0, 2114, 2115, 0, 0, 2115, 2116, 2116, 0, 2116, + 2117, 2117, 0, 0, 2117, 2118, 2118, 0, 2118, 2119, + 2119, 2120, 0, 2120, 2121, 0, 2121, 2122, 0, 0, + 2122, 2123, 2123, 0, 2123, 2124, 2124, 0, 0, 2124, + 2125, 2125, 0, 2125, 2126, 2126, 2127, 0, 2127, 2128, + 0, 2128, 2129, 0, 0, 2129, 2130, 2130, 0, 2130, + 2131, 2131, 0, 0, 2131, 2132, 2132, 0, 2132, 2133, + 2133, 2134, 0, 2134, 2135, 0, 2135, 2136, 0, 0, + 2136, 2137, 2137, 0, 2137, 2138, 2138, 0, 0, 2138, + 2139, 2139, 0, 2139, 2140, 2140, 2141, 0, 2141, 2142, + + 0, 2142, 2143, 0, 2143, 2144, 0, 2144, 2145, 2145, + 2145, 2146, 0, 2146, 2147, 2147, 2147, 0, 2147, 2147, + 2148, 0, 2148, 2149, 0, 2149, 2150, 0, 2150, 2151, + 0, 2151, 2152, 0, 2152, 2153, 0, 2153, 2154, 0, + 2154, 2155, 2155, 0, 0, 2155, 2156, 2156, 0, 2156, + 2157, 2157, 2158, 0, 2158, 2159, 0, 2159, 2160, 0, + 2160, 2161, 0, 2161, 2162, 0, 2162, 2163, 0, 2163, + 2164, 0, 2164, 2165, 0, 2165, 2166, 0, 2166, 2167, + 0, 2167, 2168, 0, 0, 2168, 2169, 2169, 0, 0, + 2169, 2170, 0, 2170, 2171, 0, 2171, 2172, 0, 2172, + + 2173, 0, 0, 2173, 2174, 0, 0, 2174, 2175, 0, + 0, 2175, 2176, 0, 0, 2176, 2177, 0, 0, 2177, + 2178, 0, 2178, 2179, 0, 2179, 2180, 0, 0, 2180, + 2181, 0, 2181, 2182, 0, 2182, 2183, 0, 2183, 2184, + 0, 2184, 2185, 0, 2185, 2186, 0, 0, 2186, 2187, + 0, 2187, 2188, 0, 2188, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, + + 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728 } ; /* The intent behind this definition is that it'll catch @@ -2703,7 +2904,7 @@ static yyconst flex_int16_t yy_chk[7486] = * We want to generate code that can be used by a reentrant parser * generated by Bison or Berkeley YACC. */ -#line 45 "scanner.l" +#line 67 "scanner.l" /* * Copyright (c) 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997 * The Regents of the University of California. All rights reserved. @@ -2723,24 +2924,8 @@ static yyconst flex_int16_t yy_chk[7486] = * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. - * - * $FreeBSD$ */ -#ifdef _WIN32 - #include <pcap-stdinc.h> -#else - #if HAVE_INTTYPES_H - #include <inttypes.h> - #elif HAVE_STDINT_H - #include <stdint.h> - #endif - #ifdef HAVE_SYS_BITYPES_H - #include <sys/bitypes.h> - #endif - #include <sys/types.h> -#endif - #include <ctype.h> #include <string.h> @@ -2760,6 +2945,8 @@ void pcap_set_column(int, yyscan_t); #ifdef INET6 #ifdef _WIN32 +#include <winsock2.h> +#include <ws2tcpip.h> /* * To quote the MSDN page for getaddrinfo() at * @@ -2779,10 +2966,9 @@ void pcap_set_column(int, yyscan_t); * Wspiapi.h header file. This inline code will be used on older Windows * platforms that do not natively support the getaddrinfo function." * - * We use getaddrinfo(), so we include Wspiapi.h here. pcap-stdinc.h - * includes Ws2tcpip.h, so we don't need to include it ourselves. + * We use getaddrinfo(), so we include Wspiapi.h here. */ -#include <Wspiapi.h> +#include <wspiapi.h> #else /* _WIN32 */ #include <sys/socket.h> /* for "struct sockaddr" in "struct addrinfo" */ #include <netdb.h> /* for "struct addrinfo" */ @@ -2805,7 +2991,12 @@ void pcap_set_column(int, yyscan_t); static int stoi(char *); static inline int xdtoi(int); -#line 2809 "scanner.c" +/* + * Disable diagnostics in the code generated by Flex. + */ +DIAG_OFF_FLEX + +#line 3000 "scanner.c" #define INITIAL 0 @@ -3038,9 +3229,9 @@ YY_DECL int yy_act; struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; -#line 242 "scanner.l" +#line 254 "scanner.l" -#line 3044 "scanner.c" +#line 3235 "scanner.c" yylval = yylval_param; @@ -3095,13 +3286,13 @@ yy_match: while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 1470 ) + if ( yy_current_state >= 1729 ) yy_c = yy_meta[(unsigned int) yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; ++yy_cp; } - while ( yy_current_state != 1469 ); + while ( yy_current_state != 1728 ); yy_cp = yyg->yy_last_accepting_cpos; yy_current_state = yyg->yy_last_accepting_state; @@ -3123,257 +3314,257 @@ do_action: /* This label is used only to access EOF actions. */ case 1: YY_RULE_SETUP -#line 243 "scanner.l" +#line 255 "scanner.l" return DST; YY_BREAK case 2: YY_RULE_SETUP -#line 244 "scanner.l" +#line 256 "scanner.l" return SRC; YY_BREAK case 3: YY_RULE_SETUP -#line 246 "scanner.l" +#line 258 "scanner.l" return LINK; YY_BREAK case 4: YY_RULE_SETUP -#line 247 "scanner.l" +#line 259 "scanner.l" return LINK; YY_BREAK case 5: YY_RULE_SETUP -#line 248 "scanner.l" +#line 260 "scanner.l" return ARP; YY_BREAK case 6: YY_RULE_SETUP -#line 249 "scanner.l" +#line 261 "scanner.l" return RARP; YY_BREAK case 7: YY_RULE_SETUP -#line 250 "scanner.l" +#line 262 "scanner.l" return IP; YY_BREAK case 8: YY_RULE_SETUP -#line 251 "scanner.l" +#line 263 "scanner.l" return SCTP; YY_BREAK case 9: YY_RULE_SETUP -#line 252 "scanner.l" +#line 264 "scanner.l" return TCP; YY_BREAK case 10: YY_RULE_SETUP -#line 253 "scanner.l" +#line 265 "scanner.l" return UDP; YY_BREAK case 11: YY_RULE_SETUP -#line 254 "scanner.l" +#line 266 "scanner.l" return ICMP; YY_BREAK case 12: YY_RULE_SETUP -#line 255 "scanner.l" +#line 267 "scanner.l" return IGMP; YY_BREAK case 13: YY_RULE_SETUP -#line 256 "scanner.l" +#line 268 "scanner.l" return IGRP; YY_BREAK case 14: YY_RULE_SETUP -#line 257 "scanner.l" +#line 269 "scanner.l" return PIM; YY_BREAK case 15: YY_RULE_SETUP -#line 258 "scanner.l" +#line 270 "scanner.l" return VRRP; YY_BREAK case 16: YY_RULE_SETUP -#line 259 "scanner.l" +#line 271 "scanner.l" return CARP; YY_BREAK case 17: YY_RULE_SETUP -#line 260 "scanner.l" +#line 272 "scanner.l" return RADIO; YY_BREAK case 18: YY_RULE_SETUP -#line 262 "scanner.l" +#line 274 "scanner.l" return IPV6; YY_BREAK case 19: YY_RULE_SETUP -#line 263 "scanner.l" +#line 275 "scanner.l" return ICMPV6; YY_BREAK case 20: YY_RULE_SETUP -#line 264 "scanner.l" +#line 276 "scanner.l" return AH; YY_BREAK case 21: YY_RULE_SETUP -#line 265 "scanner.l" +#line 277 "scanner.l" return ESP; YY_BREAK case 22: YY_RULE_SETUP -#line 267 "scanner.l" +#line 279 "scanner.l" return ATALK; YY_BREAK case 23: YY_RULE_SETUP -#line 268 "scanner.l" +#line 280 "scanner.l" return AARP; YY_BREAK case 24: YY_RULE_SETUP -#line 269 "scanner.l" +#line 281 "scanner.l" return DECNET; YY_BREAK case 25: YY_RULE_SETUP -#line 270 "scanner.l" +#line 282 "scanner.l" return LAT; YY_BREAK case 26: YY_RULE_SETUP -#line 271 "scanner.l" +#line 283 "scanner.l" return SCA; YY_BREAK case 27: YY_RULE_SETUP -#line 272 "scanner.l" +#line 284 "scanner.l" return MOPRC; YY_BREAK case 28: YY_RULE_SETUP -#line 273 "scanner.l" +#line 285 "scanner.l" return MOPDL; YY_BREAK case 29: YY_RULE_SETUP -#line 275 "scanner.l" +#line 287 "scanner.l" return ISO; YY_BREAK case 30: YY_RULE_SETUP -#line 276 "scanner.l" +#line 288 "scanner.l" return ESIS; YY_BREAK case 31: YY_RULE_SETUP -#line 277 "scanner.l" +#line 289 "scanner.l" return ESIS; YY_BREAK case 32: YY_RULE_SETUP -#line 278 "scanner.l" +#line 290 "scanner.l" return ISIS; YY_BREAK case 33: YY_RULE_SETUP -#line 279 "scanner.l" +#line 291 "scanner.l" return ISIS; YY_BREAK case 34: YY_RULE_SETUP -#line 280 "scanner.l" +#line 292 "scanner.l" return L1; YY_BREAK case 35: YY_RULE_SETUP -#line 281 "scanner.l" +#line 293 "scanner.l" return L2; YY_BREAK case 36: YY_RULE_SETUP -#line 282 "scanner.l" +#line 294 "scanner.l" return IIH; YY_BREAK case 37: YY_RULE_SETUP -#line 283 "scanner.l" +#line 295 "scanner.l" return LSP; YY_BREAK case 38: YY_RULE_SETUP -#line 284 "scanner.l" +#line 296 "scanner.l" return SNP; YY_BREAK case 39: YY_RULE_SETUP -#line 285 "scanner.l" +#line 297 "scanner.l" return CSNP; YY_BREAK case 40: YY_RULE_SETUP -#line 286 "scanner.l" +#line 298 "scanner.l" return PSNP; YY_BREAK case 41: YY_RULE_SETUP -#line 288 "scanner.l" +#line 300 "scanner.l" return CLNP; YY_BREAK case 42: YY_RULE_SETUP -#line 290 "scanner.l" +#line 302 "scanner.l" return STP; YY_BREAK case 43: YY_RULE_SETUP -#line 292 "scanner.l" +#line 304 "scanner.l" return IPX; YY_BREAK case 44: YY_RULE_SETUP -#line 294 "scanner.l" +#line 306 "scanner.l" return NETBEUI; YY_BREAK case 45: YY_RULE_SETUP -#line 296 "scanner.l" +#line 308 "scanner.l" return HOST; YY_BREAK case 46: YY_RULE_SETUP -#line 297 "scanner.l" +#line 309 "scanner.l" return NET; YY_BREAK case 47: YY_RULE_SETUP -#line 298 "scanner.l" +#line 310 "scanner.l" return NETMASK; YY_BREAK case 48: YY_RULE_SETUP -#line 299 "scanner.l" +#line 311 "scanner.l" return PORT; YY_BREAK case 49: YY_RULE_SETUP -#line 300 "scanner.l" +#line 312 "scanner.l" return PORTRANGE; YY_BREAK case 50: YY_RULE_SETUP -#line 301 "scanner.l" +#line 313 "scanner.l" return PROTO; YY_BREAK case 51: YY_RULE_SETUP -#line 302 "scanner.l" +#line 314 "scanner.l" { #ifdef NO_PROTOCHAIN bpf_error(yyextra, "%s not supported", yytext); @@ -3384,353 +3575,353 @@ YY_RULE_SETUP YY_BREAK case 52: YY_RULE_SETUP -#line 310 "scanner.l" +#line 322 "scanner.l" return GATEWAY; YY_BREAK case 53: YY_RULE_SETUP -#line 312 "scanner.l" +#line 324 "scanner.l" return TYPE; YY_BREAK case 54: YY_RULE_SETUP -#line 313 "scanner.l" +#line 325 "scanner.l" return SUBTYPE; YY_BREAK case 55: YY_RULE_SETUP -#line 314 "scanner.l" +#line 326 "scanner.l" return DIR; YY_BREAK case 56: YY_RULE_SETUP -#line 315 "scanner.l" +#line 327 "scanner.l" return ADDR1; YY_BREAK case 57: YY_RULE_SETUP -#line 316 "scanner.l" +#line 328 "scanner.l" return ADDR2; YY_BREAK case 58: YY_RULE_SETUP -#line 317 "scanner.l" +#line 329 "scanner.l" return ADDR3; YY_BREAK case 59: YY_RULE_SETUP -#line 318 "scanner.l" +#line 330 "scanner.l" return ADDR4; YY_BREAK case 60: YY_RULE_SETUP -#line 319 "scanner.l" +#line 331 "scanner.l" return RA; YY_BREAK case 61: YY_RULE_SETUP -#line 320 "scanner.l" +#line 332 "scanner.l" return TA; YY_BREAK case 62: YY_RULE_SETUP -#line 322 "scanner.l" +#line 334 "scanner.l" return LESS; YY_BREAK case 63: YY_RULE_SETUP -#line 323 "scanner.l" +#line 335 "scanner.l" return GREATER; YY_BREAK case 64: YY_RULE_SETUP -#line 324 "scanner.l" +#line 336 "scanner.l" return CBYTE; YY_BREAK case 65: YY_RULE_SETUP -#line 325 "scanner.l" +#line 337 "scanner.l" return TK_BROADCAST; YY_BREAK case 66: YY_RULE_SETUP -#line 326 "scanner.l" +#line 338 "scanner.l" return TK_MULTICAST; YY_BREAK case 67: YY_RULE_SETUP -#line 328 "scanner.l" +#line 340 "scanner.l" return AND; YY_BREAK case 68: YY_RULE_SETUP -#line 329 "scanner.l" +#line 341 "scanner.l" return OR; YY_BREAK case 69: YY_RULE_SETUP -#line 330 "scanner.l" +#line 342 "scanner.l" return '!'; YY_BREAK case 70: YY_RULE_SETUP -#line 332 "scanner.l" +#line 344 "scanner.l" return LEN; YY_BREAK case 71: YY_RULE_SETUP -#line 333 "scanner.l" +#line 345 "scanner.l" return INBOUND; YY_BREAK case 72: YY_RULE_SETUP -#line 334 "scanner.l" +#line 346 "scanner.l" return OUTBOUND; YY_BREAK case 73: YY_RULE_SETUP -#line 336 "scanner.l" +#line 348 "scanner.l" return VLAN; YY_BREAK case 74: YY_RULE_SETUP -#line 337 "scanner.l" +#line 349 "scanner.l" return MPLS; YY_BREAK case 75: YY_RULE_SETUP -#line 338 "scanner.l" +#line 350 "scanner.l" return PPPOED; YY_BREAK case 76: YY_RULE_SETUP -#line 339 "scanner.l" +#line 351 "scanner.l" return PPPOES; YY_BREAK case 77: YY_RULE_SETUP -#line 340 "scanner.l" +#line 352 "scanner.l" return GENEVE; YY_BREAK case 78: YY_RULE_SETUP -#line 342 "scanner.l" +#line 354 "scanner.l" return LANE; YY_BREAK case 79: YY_RULE_SETUP -#line 343 "scanner.l" +#line 355 "scanner.l" return LLC; YY_BREAK case 80: YY_RULE_SETUP -#line 344 "scanner.l" +#line 356 "scanner.l" return METAC; YY_BREAK case 81: YY_RULE_SETUP -#line 345 "scanner.l" +#line 357 "scanner.l" return BCC; YY_BREAK case 82: YY_RULE_SETUP -#line 346 "scanner.l" +#line 358 "scanner.l" return OAM; YY_BREAK case 83: YY_RULE_SETUP -#line 347 "scanner.l" +#line 359 "scanner.l" return OAMF4; YY_BREAK case 84: YY_RULE_SETUP -#line 348 "scanner.l" +#line 360 "scanner.l" return OAMF4EC; YY_BREAK case 85: YY_RULE_SETUP -#line 349 "scanner.l" +#line 361 "scanner.l" return OAMF4SC; YY_BREAK case 86: YY_RULE_SETUP -#line 350 "scanner.l" +#line 362 "scanner.l" return SC; YY_BREAK case 87: YY_RULE_SETUP -#line 351 "scanner.l" +#line 363 "scanner.l" return ILMIC; YY_BREAK case 88: YY_RULE_SETUP -#line 352 "scanner.l" +#line 364 "scanner.l" return VPI; YY_BREAK case 89: YY_RULE_SETUP -#line 353 "scanner.l" +#line 365 "scanner.l" return VCI; YY_BREAK case 90: YY_RULE_SETUP -#line 354 "scanner.l" +#line 366 "scanner.l" return CONNECTMSG; YY_BREAK case 91: YY_RULE_SETUP -#line 355 "scanner.l" +#line 367 "scanner.l" return METACONNECT; YY_BREAK case 92: YY_RULE_SETUP -#line 357 "scanner.l" +#line 369 "scanner.l" return PF_IFNAME; YY_BREAK case 93: YY_RULE_SETUP -#line 358 "scanner.l" +#line 370 "scanner.l" return PF_RSET; YY_BREAK case 94: YY_RULE_SETUP -#line 359 "scanner.l" +#line 371 "scanner.l" return PF_RNR; YY_BREAK case 95: YY_RULE_SETUP -#line 360 "scanner.l" +#line 372 "scanner.l" return PF_SRNR; YY_BREAK case 96: YY_RULE_SETUP -#line 361 "scanner.l" +#line 373 "scanner.l" return PF_REASON; YY_BREAK case 97: YY_RULE_SETUP -#line 362 "scanner.l" +#line 374 "scanner.l" return PF_ACTION; YY_BREAK case 98: YY_RULE_SETUP -#line 364 "scanner.l" +#line 376 "scanner.l" return FISU; YY_BREAK case 99: YY_RULE_SETUP -#line 365 "scanner.l" +#line 377 "scanner.l" return LSSU; YY_BREAK case 100: YY_RULE_SETUP -#line 366 "scanner.l" +#line 378 "scanner.l" return LSSU; YY_BREAK case 101: YY_RULE_SETUP -#line 367 "scanner.l" +#line 379 "scanner.l" return MSU; YY_BREAK case 102: YY_RULE_SETUP -#line 368 "scanner.l" +#line 380 "scanner.l" return HFISU; YY_BREAK case 103: YY_RULE_SETUP -#line 369 "scanner.l" +#line 381 "scanner.l" return HLSSU; YY_BREAK case 104: YY_RULE_SETUP -#line 370 "scanner.l" +#line 382 "scanner.l" return HMSU; YY_BREAK case 105: YY_RULE_SETUP -#line 371 "scanner.l" +#line 383 "scanner.l" return SIO; YY_BREAK case 106: YY_RULE_SETUP -#line 372 "scanner.l" +#line 384 "scanner.l" return OPC; YY_BREAK case 107: YY_RULE_SETUP -#line 373 "scanner.l" +#line 385 "scanner.l" return DPC; YY_BREAK case 108: YY_RULE_SETUP -#line 374 "scanner.l" +#line 386 "scanner.l" return SLS; YY_BREAK case 109: YY_RULE_SETUP -#line 375 "scanner.l" +#line 387 "scanner.l" return HSIO; YY_BREAK case 110: YY_RULE_SETUP -#line 376 "scanner.l" +#line 388 "scanner.l" return HOPC; YY_BREAK case 111: YY_RULE_SETUP -#line 377 "scanner.l" +#line 389 "scanner.l" return HDPC; YY_BREAK case 112: YY_RULE_SETUP -#line 378 "scanner.l" +#line 390 "scanner.l" return HSLS; YY_BREAK case 113: /* rule 113 can match eol */ YY_RULE_SETUP -#line 380 "scanner.l" +#line 392 "scanner.l" ; YY_BREAK case 114: YY_RULE_SETUP -#line 381 "scanner.l" +#line 393 "scanner.l" return yytext[0]; YY_BREAK case 115: YY_RULE_SETUP -#line 382 "scanner.l" +#line 394 "scanner.l" return GEQ; YY_BREAK case 116: YY_RULE_SETUP -#line 383 "scanner.l" +#line 395 "scanner.l" return LEQ; YY_BREAK case 117: YY_RULE_SETUP -#line 384 "scanner.l" +#line 396 "scanner.l" return NEQ; YY_BREAK case 118: YY_RULE_SETUP -#line 385 "scanner.l" +#line 397 "scanner.l" return '='; YY_BREAK case 119: YY_RULE_SETUP -#line 386 "scanner.l" +#line 398 "scanner.l" return LSH; YY_BREAK case 120: YY_RULE_SETUP -#line 387 "scanner.l" +#line 399 "scanner.l" return RSH; YY_BREAK case 121: YY_RULE_SETUP -#line 388 "scanner.l" +#line 400 "scanner.l" { yylval->e = pcap_ether_aton(((char *)yytext)+1); if (yylval->e == NULL) bpf_error(yyextra, "malloc"); @@ -3738,7 +3929,7 @@ YY_RULE_SETUP YY_BREAK case 122: YY_RULE_SETUP -#line 392 "scanner.l" +#line 404 "scanner.l" { yylval->e = pcap_ether_aton((char *)yytext); if (yylval->e == NULL) bpf_error(yyextra, "malloc"); @@ -3746,18 +3937,18 @@ YY_RULE_SETUP YY_BREAK case 123: YY_RULE_SETUP -#line 396 "scanner.l" +#line 408 "scanner.l" { yylval->i = stoi((char *)yytext); return NUM; } YY_BREAK case 124: YY_RULE_SETUP -#line 397 "scanner.l" +#line 409 "scanner.l" { yylval->s = sdup(yyextra, (char *)yytext); return HID; } YY_BREAK case 125: YY_RULE_SETUP -#line 399 "scanner.l" +#line 411 "scanner.l" { #ifdef INET6 struct addrinfo hints, *res; @@ -3777,167 +3968,302 @@ YY_RULE_SETUP YY_BREAK case 126: YY_RULE_SETUP -#line 415 "scanner.l" +#line 427 "scanner.l" { bpf_error(yyextra, "bogus ethernet address %s", yytext); } YY_BREAK case 127: YY_RULE_SETUP -#line 416 "scanner.l" +#line 428 "scanner.l" { yylval->i = 0; return NUM; } YY_BREAK case 128: YY_RULE_SETUP -#line 417 "scanner.l" +#line 429 "scanner.l" { yylval->i = 1; return NUM; } YY_BREAK case 129: YY_RULE_SETUP -#line 418 "scanner.l" +#line 430 "scanner.l" { yylval->i = 0; return NUM; } YY_BREAK case 130: YY_RULE_SETUP -#line 419 "scanner.l" +#line 431 "scanner.l" { yylval->i = 3; return NUM; } YY_BREAK case 131: YY_RULE_SETUP -#line 420 "scanner.l" +#line 432 "scanner.l" { yylval->i = 4; return NUM; } YY_BREAK case 132: YY_RULE_SETUP -#line 421 "scanner.l" +#line 433 "scanner.l" { yylval->i = 5; return NUM; } YY_BREAK case 133: YY_RULE_SETUP -#line 422 "scanner.l" +#line 434 "scanner.l" { yylval->i = 8; return NUM; } YY_BREAK case 134: YY_RULE_SETUP -#line 423 "scanner.l" +#line 435 "scanner.l" { yylval->i = 9; return NUM; } YY_BREAK case 135: YY_RULE_SETUP -#line 424 "scanner.l" +#line 436 "scanner.l" { yylval->i = 10; return NUM; } YY_BREAK case 136: YY_RULE_SETUP -#line 425 "scanner.l" +#line 437 "scanner.l" { yylval->i = 11; return NUM; } YY_BREAK case 137: YY_RULE_SETUP -#line 426 "scanner.l" +#line 438 "scanner.l" { yylval->i = 12; return NUM; } YY_BREAK case 138: YY_RULE_SETUP -#line 427 "scanner.l" +#line 439 "scanner.l" { yylval->i = 13; return NUM; } YY_BREAK case 139: YY_RULE_SETUP -#line 428 "scanner.l" +#line 440 "scanner.l" { yylval->i = 14; return NUM; } YY_BREAK case 140: YY_RULE_SETUP -#line 429 "scanner.l" +#line 441 "scanner.l" { yylval->i = 15; return NUM; } YY_BREAK case 141: YY_RULE_SETUP -#line 430 "scanner.l" +#line 442 "scanner.l" { yylval->i = 16; return NUM; } YY_BREAK case 142: YY_RULE_SETUP -#line 431 "scanner.l" +#line 443 "scanner.l" { yylval->i = 17; return NUM; } YY_BREAK case 143: YY_RULE_SETUP -#line 432 "scanner.l" +#line 444 "scanner.l" { yylval->i = 18; return NUM; } YY_BREAK case 144: YY_RULE_SETUP -#line 433 "scanner.l" -{ yylval->i = 13; return NUM; } +#line 446 "scanner.l" +{ yylval->i = 0; return NUM; } YY_BREAK case 145: YY_RULE_SETUP -#line 434 "scanner.l" -{ yylval->i = 0x01; return NUM; } +#line 447 "scanner.l" +{ yylval->i = 1; return NUM; } YY_BREAK case 146: YY_RULE_SETUP -#line 435 "scanner.l" -{ yylval->i = 0x02; return NUM; } +#line 449 "scanner.l" +{ yylval->i = 128; return NUM; } YY_BREAK case 147: YY_RULE_SETUP -#line 436 "scanner.l" -{ yylval->i = 0x04; return NUM; } +#line 450 "scanner.l" +{ yylval->i = 129; return NUM; } YY_BREAK case 148: YY_RULE_SETUP -#line 437 "scanner.l" -{ yylval->i = 0x08; return NUM; } +#line 451 "scanner.l" +{ yylval->i = 130; return NUM; } YY_BREAK case 149: YY_RULE_SETUP -#line 438 "scanner.l" -{ yylval->i = 0x10; return NUM; } +#line 452 "scanner.l" +{ yylval->i = 131; return NUM; } YY_BREAK case 150: YY_RULE_SETUP -#line 439 "scanner.l" -{ yylval->i = 0x20; return NUM; } +#line 453 "scanner.l" +{ yylval->i = 132; return NUM; } YY_BREAK case 151: YY_RULE_SETUP -#line 440 "scanner.l" -{ yylval->i = 0x40; return NUM; } +#line 454 "scanner.l" +{ yylval->i = 133; return NUM; } YY_BREAK case 152: YY_RULE_SETUP -#line 441 "scanner.l" -{ yylval->i = 0x80; return NUM; } +#line 455 "scanner.l" +{ yylval->i = 134; return NUM; } YY_BREAK case 153: YY_RULE_SETUP -#line 442 "scanner.l" +#line 456 "scanner.l" +{ yylval->i = 135; return NUM; } + YY_BREAK +case 154: +YY_RULE_SETUP +#line 457 "scanner.l" +{ yylval->i = 136; return NUM; } + YY_BREAK +case 155: +YY_RULE_SETUP +#line 458 "scanner.l" +{ yylval->i = 137; return NUM; } + YY_BREAK +case 156: +YY_RULE_SETUP +#line 459 "scanner.l" +{ yylval->i = 138; return NUM; } + YY_BREAK +case 157: +YY_RULE_SETUP +#line 460 "scanner.l" +{ yylval->i = 139; return NUM; } + YY_BREAK +case 158: +YY_RULE_SETUP +#line 461 "scanner.l" +{ yylval->i = 140; return NUM; } + YY_BREAK +case 159: +YY_RULE_SETUP +#line 462 "scanner.l" +{ yylval->i = 141; return NUM; } + YY_BREAK +case 160: +YY_RULE_SETUP +#line 463 "scanner.l" +{ yylval->i = 142; return NUM; } + YY_BREAK +case 161: +YY_RULE_SETUP +#line 464 "scanner.l" +{ yylval->i = 143; return NUM; } + YY_BREAK +case 162: +YY_RULE_SETUP +#line 465 "scanner.l" +{ yylval->i = 144; return NUM; } + YY_BREAK +case 163: +YY_RULE_SETUP +#line 466 "scanner.l" +{ yylval->i = 145; return NUM; } + YY_BREAK +case 164: +YY_RULE_SETUP +#line 467 "scanner.l" +{ yylval->i = 146; return NUM; } + YY_BREAK +case 165: +YY_RULE_SETUP +#line 468 "scanner.l" +{ yylval->i = 147; return NUM; } + YY_BREAK +case 166: +YY_RULE_SETUP +#line 469 "scanner.l" +{ yylval->i = 148; return NUM; } + YY_BREAK +case 167: +YY_RULE_SETUP +#line 470 "scanner.l" +{ yylval->i = 149; return NUM; } + YY_BREAK +case 168: +YY_RULE_SETUP +#line 471 "scanner.l" +{ yylval->i = 151; return NUM; } + YY_BREAK +case 169: +YY_RULE_SETUP +#line 472 "scanner.l" +{ yylval->i = 152; return NUM; } + YY_BREAK +case 170: +YY_RULE_SETUP +#line 473 "scanner.l" +{ yylval->i = 153; return NUM; } + YY_BREAK +case 171: +YY_RULE_SETUP +#line 475 "scanner.l" +{ yylval->i = 13; return NUM; } + YY_BREAK +case 172: +YY_RULE_SETUP +#line 476 "scanner.l" +{ yylval->i = 0x01; return NUM; } + YY_BREAK +case 173: +YY_RULE_SETUP +#line 477 "scanner.l" +{ yylval->i = 0x02; return NUM; } + YY_BREAK +case 174: +YY_RULE_SETUP +#line 478 "scanner.l" +{ yylval->i = 0x04; return NUM; } + YY_BREAK +case 175: +YY_RULE_SETUP +#line 479 "scanner.l" +{ yylval->i = 0x08; return NUM; } + YY_BREAK +case 176: +YY_RULE_SETUP +#line 480 "scanner.l" +{ yylval->i = 0x10; return NUM; } + YY_BREAK +case 177: +YY_RULE_SETUP +#line 481 "scanner.l" +{ yylval->i = 0x20; return NUM; } + YY_BREAK +case 178: +YY_RULE_SETUP +#line 482 "scanner.l" +{ yylval->i = 0x40; return NUM; } + YY_BREAK +case 179: +YY_RULE_SETUP +#line 483 "scanner.l" +{ yylval->i = 0x80; return NUM; } + YY_BREAK +case 180: +YY_RULE_SETUP +#line 484 "scanner.l" { yylval->s = sdup(yyextra, (char *)yytext); return ID; } YY_BREAK -case 154: +case 181: YY_RULE_SETUP -#line 444 "scanner.l" +#line 486 "scanner.l" { yylval->s = sdup(yyextra, (char *)yytext + 1); return ID; } YY_BREAK -case 155: +case 182: YY_RULE_SETUP -#line 445 "scanner.l" +#line 487 "scanner.l" { bpf_error(yyextra, "illegal token: %s", yytext); } YY_BREAK -case 156: +case 183: YY_RULE_SETUP -#line 447 "scanner.l" +#line 489 "scanner.l" { bpf_error(yyextra, "illegal char '%c'", *yytext); } YY_BREAK -case 157: +case 184: YY_RULE_SETUP -#line 448 "scanner.l" +#line 490 "scanner.l" ECHO; YY_BREAK -#line 3941 "scanner.c" +#line 4267 "scanner.c" case YY_STATE_EOF(INITIAL): yyterminate(); @@ -4232,7 +4558,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 1470 ) + if ( yy_current_state >= 1729 ) yy_c = yy_meta[(unsigned int) yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; @@ -4261,11 +4587,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 1470 ) + if ( yy_current_state >= 1729 ) yy_c = yy_meta[(unsigned int) yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; - yy_is_jam = (yy_current_state == 1469); + yy_is_jam = (yy_current_state == 1728); (void)yyg; return yy_is_jam ? 0 : yy_current_state; @@ -5064,14 +5390,18 @@ void pcap_free (void * ptr , yyscan_t yyscanner) #define YYTABLES_NAME "yytables" -#line 448 "scanner.l" +#line 490 "scanner.l" + +/* + * Turn diagnostics back on, so we check the code that we've written. + */ +DIAG_ON_FLEX /* Hex digit to integer. */ static inline int -xdtoi(c) - register int c; +xdtoi(int c) { if (isdigit(c)) return c - '0'; @@ -5086,8 +5416,7 @@ xdtoi(c) * preceding 0x or 0 and uses hex or octal instead of decimal. */ static int -stoi(s) - char *s; +stoi(char *s) { int base = 10; int n = 0; diff --git a/freebsd/contrib/libpcap/scanner.h b/freebsd/contrib/libpcap/scanner.h index 1aea2cc7..48ed6ee4 100644 --- a/freebsd/contrib/libpcap/scanner.h +++ b/freebsd/contrib/libpcap/scanner.h @@ -6,12 +6,34 @@ #line 2 "scanner.l" /* Must come first for _LARGE_FILE_API on AIX. */ #ifdef HAVE_CONFIG_H -#include "config.h" +#include <config.h> #endif +/* + * Must come first to avoid warnings on Windows. + * + * Flex-generated scanners may only include <inttypes.h> if __STDC_VERSION__ + * is defined with a value >= 199901, meaning "full C99", and MSVC may not + * define it with that value, because it isn't 100% C99-compliant, even + * though it has an <inttypes.h> capable of defining everything the Flex + * scanner needs. + * + * We, however, will include it if we know we have an MSVC version that has + * it; this means that we may define the INTn_MAX and UINTn_MAX values in + * scanner.c, and then include <stdint.h>, which may define them differently + * (same value, but different string of characters), causing compiler warnings. + * + * If we include it here, and they're defined, that'll prevent scanner.c + * from defining them. So we include <pcap/pcap-inttypes.h>, to get + * <inttypes.h> if we have it. + */ +#include <pcap/pcap-inttypes.h> + +#include "diag-control.h" + -#line 15 "scanner.h" +#line 37 "scanner.h" #define YY_INT_ALIGNED short int @@ -354,9 +376,9 @@ extern int pcap_lex \ #undef YY_DECL #endif -#line 448 "scanner.l" +#line 490 "scanner.l" -#line 361 "scanner.h" +#line 383 "scanner.h" #undef pcap_IN_HEADER #endif /* pcap_HEADER_H */ diff --git a/freebsd/contrib/libpcap/scanner.l b/freebsd/contrib/libpcap/scanner.l index d71a9bed..e0890b43 100644 --- a/freebsd/contrib/libpcap/scanner.l +++ b/freebsd/contrib/libpcap/scanner.l @@ -1,8 +1,30 @@ %top { /* Must come first for _LARGE_FILE_API on AIX. */ #ifdef HAVE_CONFIG_H -#include "config.h" +#include <config.h> #endif + +/* + * Must come first to avoid warnings on Windows. + * + * Flex-generated scanners may only include <inttypes.h> if __STDC_VERSION__ + * is defined with a value >= 199901, meaning "full C99", and MSVC may not + * define it with that value, because it isn't 100% C99-compliant, even + * though it has an <inttypes.h> capable of defining everything the Flex + * scanner needs. + * + * We, however, will include it if we know we have an MSVC version that has + * it; this means that we may define the INTn_MAX and UINTn_MAX values in + * scanner.c, and then include <stdint.h>, which may define them differently + * (same value, but different string of characters), causing compiler warnings. + * + * If we include it here, and they're defined, that'll prevent scanner.c + * from defining them. So we include <pcap/pcap-inttypes.h>, to get + * <inttypes.h> if we have it. + */ +#include <pcap/pcap-inttypes.h> + +#include "diag-control.h" } /* @@ -61,24 +83,8 @@ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. - * - * $FreeBSD$ */ -#ifdef _WIN32 - #include <pcap-stdinc.h> -#else - #if HAVE_INTTYPES_H - #include <inttypes.h> - #elif HAVE_STDINT_H - #include <stdint.h> - #endif - #ifdef HAVE_SYS_BITYPES_H - #include <sys/bitypes.h> - #endif - #include <sys/types.h> -#endif - #include <ctype.h> #include <string.h> @@ -98,6 +104,8 @@ void pcap_set_column(int, yyscan_t); #ifdef INET6 #ifdef _WIN32 +#include <winsock2.h> +#include <ws2tcpip.h> /* * To quote the MSDN page for getaddrinfo() at * @@ -117,10 +125,9 @@ void pcap_set_column(int, yyscan_t); * Wspiapi.h header file. This inline code will be used on older Windows * platforms that do not natively support the getaddrinfo function." * - * We use getaddrinfo(), so we include Wspiapi.h here. pcap-stdinc.h - * includes Ws2tcpip.h, so we don't need to include it ourselves. + * We use getaddrinfo(), so we include Wspiapi.h here. */ -#include <Wspiapi.h> +#include <wspiapi.h> #else /* _WIN32 */ #include <sys/socket.h> /* for "struct sockaddr" in "struct addrinfo" */ #include <netdb.h> /* for "struct addrinfo" */ @@ -143,6 +150,11 @@ void pcap_set_column(int, yyscan_t); static int stoi(char *); static inline int xdtoi(int); +/* + * Disable diagnostics in the code generated by Flex. + */ +DIAG_OFF_FLEX + %} N ([0-9]+|(0X|0x)[0-9A-Fa-f]+) @@ -430,6 +442,36 @@ icmp-ireq { yylval->i = 15; return NUM; } icmp-ireqreply { yylval->i = 16; return NUM; } icmp-maskreq { yylval->i = 17; return NUM; } icmp-maskreply { yylval->i = 18; return NUM; } + +icmp6type { yylval->i = 0; return NUM; } +icmp6code { yylval->i = 1; return NUM; } + +icmp6-echo { yylval->i = 128; return NUM; } +icmp6-echoreply { yylval->i = 129; return NUM; } +icmp6-multicastlistenerquery { yylval->i = 130; return NUM; } +icmp6-multicastlistenerreportv1 { yylval->i = 131; return NUM; } +icmp6-multicastlistenerdone { yylval->i = 132; return NUM; } +icmp6-routersolicit { yylval->i = 133; return NUM; } +icmp6-routeradvert { yylval->i = 134; return NUM; } +icmp6-neighborsolicit { yylval->i = 135; return NUM; } +icmp6-neighboradvert { yylval->i = 136; return NUM; } +icmp6-redirect { yylval->i = 137; return NUM; } +icmp6-routerrenum { yylval->i = 138; return NUM; } +icmp6-nodeinformationquery { yylval->i = 139; return NUM; } +icmp6-nodeinformationresponse { yylval->i = 140; return NUM; } +icmp6-ineighbordiscoverysolicit { yylval->i = 141; return NUM; } +icmp6-ineighbordiscoveryadvert { yylval->i = 142; return NUM; } +icmp6-multicastlistenerreportv2 { yylval->i = 143; return NUM; } +icmp6-homeagentdiscoveryrequest { yylval->i = 144; return NUM; } +icmp6-homeagentdiscoveryreply { yylval->i = 145; return NUM; } +icmp6-mobileprefixsolicit { yylval->i = 146; return NUM; } +icmp6-mobileprefixadvert { yylval->i = 147; return NUM; } +icmp6-certpathsolicit { yylval->i = 148; return NUM; } +icmp6-certpathadvert { yylval->i = 149; return NUM; } +icmp6-multicastrouteradvert { yylval->i = 151; return NUM; } +icmp6-multicastroutersolicit { yylval->i = 152; return NUM; } +icmp6-multicastrouterterm { yylval->i = 153; return NUM; } + tcpflags { yylval->i = 13; return NUM; } tcp-fin { yylval->i = 0x01; return NUM; } tcp-syn { yylval->i = 0x02; return NUM; } @@ -447,10 +489,14 @@ tcp-cwr { yylval->i = 0x80; return NUM; } . { bpf_error(yyextra, "illegal char '%c'", *yytext); } %% +/* + * Turn diagnostics back on, so we check the code that we've written. + */ +DIAG_ON_FLEX + /* Hex digit to integer. */ static inline int -xdtoi(c) - register int c; +xdtoi(int c) { if (isdigit(c)) return c - '0'; @@ -465,8 +511,7 @@ xdtoi(c) * preceding 0x or 0 and uses hex or octal instead of decimal. */ static int -stoi(s) - char *s; +stoi(char *s) { int base = 10; int n = 0; diff --git a/freebsd/contrib/libpcap/sf-pcap.c b/freebsd/contrib/libpcap/sf-pcap.c index d7cb806f..c9ac143d 100644 --- a/freebsd/contrib/libpcap/sf-pcap.c +++ b/freebsd/contrib/libpcap/sf-pcap.c @@ -30,27 +30,14 @@ * dependent values so we can print the dump file on any architecture. */ -#ifndef lint -static const char rcsid[] _U_ = - "@(#) $Header$ (LBL)"; -#endif - #ifdef HAVE_CONFIG_H -#include "config.h" +#include <config.h> #endif +#include <pcap-types.h> #ifdef _WIN32 -#include <pcap-stdinc.h> -#else /* _WIN32 */ -#if HAVE_INTTYPES_H -#include <inttypes.h> -#elif HAVE_STDINT_H -#include <stdint.h> -#endif -#ifdef HAVE_SYS_BITYPES_H -#include <sys/bitypes.h> -#endif -#include <sys/types.h> +#include <io.h> +#include <fcntl.h> #endif /* _WIN32 */ #include <errno.h> @@ -187,9 +174,8 @@ pcap_check_header(bpf_u_int32 magic, FILE *fp, u_int precision, char *errbuf, sizeof(hdr) - sizeof(hdr.magic), fp); if (amt_read != sizeof(hdr) - sizeof(hdr.magic)) { if (ferror(fp)) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "error reading dump file: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "error reading dump file"); } else { pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, "truncated dump file; tried to read %lu file header bytes, only got %lu", @@ -234,14 +220,6 @@ pcap_check_header(bpf_u_int32 magic, FILE *fp, u_int precision, char *errbuf, return NULL; } - if (hdr.snaplen > MAXIMUM_SNAPLEN) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "invalid file capture length %u, bigger than " - "maximum of %u", hdr.snaplen, MAXIMUM_SNAPLEN); - *err = 1; - return NULL; - } - /* * OK, this is a good pcap file. * Allocate a pcap_t for it. @@ -257,6 +235,17 @@ pcap_check_header(bpf_u_int32 magic, FILE *fp, u_int precision, char *errbuf, p->version_minor = hdr.version_minor; p->tzoff = hdr.thiszone; p->snapshot = hdr.snaplen; + if (p->snapshot <= 0) { + /* + * Bogus snapshot length; use the maximum for this + * link-layer type as a fallback. + * + * XXX - the only reason why snapshot is signed is + * that pcap_snapshot() returns an int, not an + * unsigned int. + */ + p->snapshot = max_snaplen_for_dlt(hdr.linktype); + } p->linktype = linktype_to_dlt(LT_LINKTYPE(hdr.linktype)); p->linktype_ext = LT_LINKTYPE_EXT(hdr.linktype); @@ -392,14 +381,16 @@ pcap_check_header(bpf_u_int32 magic, FILE *fp, u_int precision, char *errbuf, /* * Allocate a buffer for the packet data. + * Choose the minimum of the file's snapshot length and 2K bytes; + * that should be enough for most network packets - we'll grow it + * if necessary. That way, we don't allocate a huge chunk of + * memory just because there's a huge snapshot length, as the + * snapshot length might be larger than the size of the largest + * packet. */ p->bufsize = p->snapshot; - if (p->bufsize <= 0) { - /* - * Bogus snapshot length; use the maximum as a fallback. - */ - p->bufsize = MAXIMUM_SNAPLEN; - } + if (p->bufsize > 2048) + p->bufsize = 2048; p->buffer = malloc(p->bufsize); if (p->buffer == NULL) { pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, "out of memory"); @@ -414,6 +405,24 @@ pcap_check_header(bpf_u_int32 magic, FILE *fp, u_int precision, char *errbuf, } /* + * Grow the packet buffer to the specified size. + */ +static int +grow_buffer(pcap_t *p, u_int bufsize) +{ + void *bigger_buffer; + + bigger_buffer = realloc(p->buffer, bufsize); + if (bigger_buffer == NULL) { + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "out of memory"); + return (0); + } + p->buffer = bigger_buffer; + p->bufsize = bufsize; + return (1); +} + +/* * Read and return the next packet from the savefile. Return the header * in hdr and a pointer to the contents in data. Return 0 on success, 1 * if there were no more packets, and -1 on an error. @@ -437,9 +446,8 @@ pcap_next_packet(pcap_t *p, struct pcap_pkthdr *hdr, u_char **data) amt_read = fread(&sf_hdr, 1, ps->hdrsize, fp); if (amt_read != ps->hdrsize) { if (ferror(fp)) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "error reading dump file: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "error reading dump file"); return (-1); } else { if (amt_read != 0) { @@ -515,41 +523,87 @@ pcap_next_packet(pcap_t *p, struct pcap_pkthdr *hdr, u_char **data) break; } - if (hdr->caplen > p->bufsize) { + /* + * Is the packet bigger than we consider sane? + */ + if (hdr->caplen > max_snaplen_for_dlt(p->linktype)) { + /* + * Yes. This may be a damaged or fuzzed file. + * + * Is it bigger than the snapshot length? + * (We don't treat that as an error if it's not + * bigger than the maximum we consider sane; see + * below.) + */ + if (hdr->caplen > (bpf_u_int32)p->snapshot) { + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "invalid packet capture length %u, bigger than " + "snaplen of %d", hdr->caplen, p->snapshot); + } else { + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "invalid packet capture length %u, bigger than " + "maximum of %u", hdr->caplen, + max_snaplen_for_dlt(p->linktype)); + } + return (-1); + } + + if (hdr->caplen > (bpf_u_int32)p->snapshot) { /* + * The packet is bigger than the snapshot length + * for this file. + * * This can happen due to Solaris 2.3 systems tripping * over the BUFMOD problem and not setting the snapshot - * correctly in the savefile header. - * This can also happen with a corrupted savefile or a - * savefile built/modified by a fuzz tester. - * If the caplen isn't grossly wrong, try to salvage. + * length correctly in the savefile header. + * + * libpcap 0.4 and later on Solaris 2.3 should set the + * snapshot length correctly in the pcap file header, + * even though they don't set a snapshot length in bufmod + * (the buggy bufmod chops off the *beginning* of the + * packet if a snapshot length is specified); they should + * also reduce the captured length, as supplied to the + * per-packet callback, to the snapshot length if it's + * greater than the snapshot length, so the code using + * libpcap should see the packet cut off at the snapshot + * length, even though the full packet is copied up to + * userland. + * + * However, perhaps some versions of libpcap failed to + * set the snapshot length currectly in the file header + * or the per-packet header, or perhaps this is a + * corrupted safefile or a savefile built/modified by a + * fuzz tester, so we check anyway. We grow the buffer + * to be big enough for the snapshot length, read up + * to the snapshot length, discard the rest of the + * packet, and report the snapshot length as the captured + * length; we don't want to hand our caller a packet + * bigger than the snapshot length, because they might + * be assuming they'll never be handed such a packet, + * and might copy the packet into a snapshot-length- + * sized buffer, assuming it'll fit. */ size_t bytes_to_discard; size_t bytes_to_read, bytes_read; char discard_buf[4096]; - if (hdr->caplen > MAXIMUM_SNAPLEN) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "invalid packet capture length %u, bigger than " - "maximum of %u", hdr->caplen, MAXIMUM_SNAPLEN); - return (-1); + if (hdr->caplen > p->bufsize) { + /* + * Grow the buffer to the snapshot length. + */ + if (!grow_buffer(p, p->snapshot)) + return (-1); } /* - * XXX - we don't grow the buffer here because some - * program might assume that it will never get packets - * bigger than the snapshot length; for example, it might - * copy data from our buffer to a buffer of its own, - * allocated based on the return value of pcap_snapshot(). - * - * Read the first p->bufsize bytes into the buffer. + * Read the first p->snapshot bytes into the buffer. */ - amt_read = fread(p->buffer, 1, p->bufsize, fp); - if (amt_read != p->bufsize) { + amt_read = fread(p->buffer, 1, p->snapshot, fp); + if (amt_read != (bpf_u_int32)p->snapshot) { if (ferror(fp)) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "error reading dump file: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, + PCAP_ERRBUF_SIZE, errno, + "error reading dump file"); } else { /* * Yes, this uses hdr->caplen; technically, @@ -560,7 +614,7 @@ pcap_next_packet(pcap_t *p, struct pcap_pkthdr *hdr, u_char **data) */ pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "truncated dump file; tried to read %u captured bytes, only got %lu", - hdr->caplen, (unsigned long)amt_read); + p->snapshot, (unsigned long)amt_read); } return (-1); } @@ -568,7 +622,7 @@ pcap_next_packet(pcap_t *p, struct pcap_pkthdr *hdr, u_char **data) /* * Now read and discard what's left. */ - bytes_to_discard = hdr->caplen - p->bufsize; + bytes_to_discard = hdr->caplen - p->snapshot; bytes_read = amt_read; while (bytes_to_discard != 0) { bytes_to_read = bytes_to_discard; @@ -578,9 +632,9 @@ pcap_next_packet(pcap_t *p, struct pcap_pkthdr *hdr, u_char **data) bytes_read += amt_read; if (amt_read != bytes_to_read) { if (ferror(fp)) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "error reading dump file: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, + PCAP_ERRBUF_SIZE, errno, + "error reading dump file"); } else { pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "truncated dump file; tried to read %u captured bytes, only got %lu", @@ -595,15 +649,41 @@ pcap_next_packet(pcap_t *p, struct pcap_pkthdr *hdr, u_char **data) * Adjust caplen accordingly, so we don't get confused later * as to how many bytes we have to play with. */ - hdr->caplen = p->bufsize; + hdr->caplen = p->snapshot; } else { + if (hdr->caplen > p->bufsize) { + /* + * Grow the buffer to the next power of 2, or + * the snaplen, whichever is lower. + */ + u_int new_bufsize; + + new_bufsize = hdr->caplen; + /* + * http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 + */ + new_bufsize--; + new_bufsize |= new_bufsize >> 1; + new_bufsize |= new_bufsize >> 2; + new_bufsize |= new_bufsize >> 4; + new_bufsize |= new_bufsize >> 8; + new_bufsize |= new_bufsize >> 16; + new_bufsize++; + + if (new_bufsize > (u_int)p->snapshot) + new_bufsize = p->snapshot; + + if (!grow_buffer(p, new_bufsize)) + return (-1); + } + /* read the packet itself */ amt_read = fread(p->buffer, 1, hdr->caplen, fp); if (amt_read != hdr->caplen) { if (ferror(fp)) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "error reading dump file: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, + PCAP_ERRBUF_SIZE, errno, + "error reading dump file"); } else { pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "truncated dump file; tried to read %u captured bytes, only got %lu", @@ -674,11 +754,11 @@ pcap_setup_dump(pcap_t *p, int linktype, FILE *f, const char *fname) if (f == stdout) SET_BINMODE(f); else - setbuf(f, NULL); + setvbuf(f, NULL, _IONBF, 0); #endif if (sf_write_header(p, f, linktype, p->tzoff, p->snapshot) == -1) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "Can't write to %s: %s", - fname, pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "Can't write to %s", fname); if (f != stdout) (void)fclose(f); return (NULL); @@ -723,14 +803,16 @@ pcap_dump_open(pcap_t *p, const char *fname) f = stdout; fname = "standard output"; } else { -#if !defined(_WIN32) && !defined(MSDOS) - f = fopen(fname, "w"); -#else + /* + * "b" is supported as of C90, so *all* UN*Xes should + * support it, even though it does nothing. It's + * required on Windows, as the file is a binary file + * and must be written in binary mode. + */ f = fopen(fname, "wb"); -#endif if (f == NULL) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "%s: %s", - fname, pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "%s", fname); return (NULL); } } @@ -781,14 +863,15 @@ pcap_dump_open_append(pcap_t *p, const char *fname) if (fname[0] == '-' && fname[1] == '\0') return (pcap_setup_dump(p, linktype, stdout, "standard output")); -#if !defined(_WIN32) && !defined(MSDOS) - f = fopen(fname, "r+"); -#else + /* + * "b" is supported as of C90, so *all* UN*Xes should support it, + * even though it does nothing. It's required on Windows, as the + * file is a binary file and must be read in binary mode. + */ f = fopen(fname, "rb+"); -#endif if (f == NULL) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "%s: %s", - fname, pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "%s", fname); return (NULL); } @@ -798,8 +881,8 @@ pcap_dump_open_append(pcap_t *p, const char *fname) amt_read = fread(&ph, 1, sizeof (ph), f); if (amt_read != sizeof (ph)) { if (ferror(f)) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "%s: %s", - fname, pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "%s", fname); fclose(f); return (NULL); } else if (feof(f) && amt_read > 0) { @@ -815,7 +898,7 @@ pcap_dump_open_append(pcap_t *p, const char *fname) * We turn off buffering. * XXX - why? And why not on the standard output? */ - setbuf(f, NULL); + setvbuf(f, NULL, _IONBF, 0); #endif /* @@ -906,8 +989,8 @@ pcap_dump_open_append(pcap_t *p, const char *fname) * A header isn't present; attempt to write it. */ if (sf_write_header(p, f, linktype, p->tzoff, p->snapshot) == -1) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "Can't write to %s: %s", - fname, pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "Can't write to %s", fname); (void)fclose(f); return (NULL); } @@ -917,8 +1000,8 @@ pcap_dump_open_append(pcap_t *p, const char *fname) * Start writing at the end of the file. */ if (fseek(f, 0, SEEK_END) == -1) { - pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, "Can't seek to end of %s: %s", - fname, pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE, + errno, "Can't seek to end of %s", fname); (void)fclose(f); return (NULL); } @@ -937,6 +1020,45 @@ pcap_dump_ftell(pcap_dumper_t *p) return (ftell((FILE *)p)); } +#if defined(HAVE_FSEEKO) +/* + * We have fseeko(), so we have ftello(). + * If we have large file support (files larger than 2^31-1 bytes), + * ftello() will give us a current file position with more than 32 + * bits. + */ +int64_t +pcap_dump_ftell64(pcap_dumper_t *p) +{ + return (ftello((FILE *)p)); +} +#elif defined(_MSC_VER) +/* + * We have Visual Studio; we support only 2005 and later, so we have + * _ftelli64(). + */ +int64_t +pcap_dump_ftell64(pcap_dumper_t *p) +{ + return (_ftelli64((FILE *)p)); +} +#else +/* + * We don't have ftello() or _ftelli64(), so fall back on ftell(). + * Either long is 64 bits, in which case ftell() should suffice, + * or this is probably an older 32-bit UN*X without large file + * support, which means you'll probably get errors trying to + * write files > 2^31-1, so it won't matter anyway. + * + * XXX - what about MinGW? + */ +int64_t +pcap_dump_ftell64(pcap_dumper_t *p) +{ + return (ftell((FILE *)p)); +} +#endif + int pcap_dump_flush(pcap_dumper_t *p) { diff --git a/freebsd/contrib/libpcap/sf-pcap-ng.c b/freebsd/contrib/libpcap/sf-pcapng.c index 209c1424..f5719042 100644 --- a/freebsd/contrib/libpcap/sf-pcap-ng.c +++ b/freebsd/contrib/libpcap/sf-pcapng.c @@ -20,31 +20,14 @@ * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. * - * sf-pcap-ng.c - pcap-ng-file-format-specific code from savefile.c + * sf-pcapng.c - pcapng-file-format-specific code from savefile.c */ -#ifndef lint -static const char rcsid[] _U_ = - "@(#) $Header$ (LBL)"; -#endif - #ifdef HAVE_CONFIG_H -#include "config.h" +#include <config.h> #endif -#ifdef _WIN32 -#include <pcap-stdinc.h> -#else /* _WIN32 */ -#if HAVE_INTTYPES_H -#include <inttypes.h> -#elif HAVE_STDINT_H -#include <stdint.h> -#endif -#ifdef HAVE_SYS_BITYPES_H -#include <sys/bitypes.h> -#endif -#include <sys/types.h> -#endif /* _WIN32 */ +#include <pcap/pcap-inttypes.h> #include <errno.h> #include <memory.h> @@ -60,7 +43,7 @@ static const char rcsid[] _U_ = #include "os-proto.h" #endif -#include "sf-pcap-ng.h" +#include "sf-pcapng.h" /* * Block types. @@ -109,7 +92,7 @@ struct section_header_block { bpf_u_int32 byte_order_magic; u_short major_version; u_short minor_version; - u_int64_t section_length; + uint64_t section_length; /* followed by options and trailer */ }; @@ -219,16 +202,48 @@ struct pcap_ng_if { u_int tsresol; /* time stamp resolution */ tstamp_scale_type_t scale_type; /* how to scale */ u_int scale_factor; /* time stamp scale factor for power-of-10 tsresol */ - u_int64_t tsoffset; /* time stamp offset */ + uint64_t tsoffset; /* time stamp offset */ }; +/* + * Per-pcap_t private data. + * + * max_blocksize is the maximum size of a block that we'll accept. We + * reject blocks bigger than this, so we don't consume too much memory + * with a truly huge block. It can change as we see IDBs with different + * link-layer header types. (Currently, we don't support IDBs with + * different link-layer header types, but we will support it in the + * future, when we offer file-reading APIs that support it.) + * + * XXX - that's an issue on ILP32 platforms, where the maximum block + * size of 2^31-1 would eat all but one byte of the entire address space. + * It's less of an issue on ILP64/LLP64 platforms, but the actual size + * of the address space may be limited by 1) the number of *significant* + * address bits (currently, x86-64 only supports 48 bits of address), 2) + * any limitations imposed by the operating system; 3) any limitations + * imposed by the amount of available backing store for anonymous pages, + * so we impose a limit regardless of the size of a pointer. + */ struct pcap_ng_sf { u_int user_tsresol; /* time stamp resolution requested by the user */ + u_int max_blocksize; /* don't grow buffer size past this */ bpf_u_int32 ifcount; /* number of interfaces seen in this capture */ bpf_u_int32 ifaces_size; /* size of array below */ struct pcap_ng_if *ifaces; /* array of interface information */ }; +/* + * Maximum block size for a given maximum snapshot length; we calculate + * this based + * + * We define it as the size of an EPB with a max_snaplen-sized + * packet and 128KB of options. + */ +#define MAX_BLOCKSIZE(max_snaplen) (sizeof (struct block_header) + \ + sizeof (struct enhanced_packet_block) + \ + (max_snaplen) + 131072 + \ + sizeof (struct block_trailer)) + static void pcap_ng_cleanup(pcap_t *p); static int pcap_ng_next_packet(pcap_t *p, struct pcap_pkthdr *hdr, u_char **data); @@ -242,9 +257,8 @@ read_bytes(FILE *fp, void *buf, size_t bytes_to_read, int fail_on_eof, amt_read = fread(buf, 1, bytes_to_read, fp); if (amt_read != bytes_to_read) { if (ferror(fp)) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "error reading dump file: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "error reading dump file"); } else { if (amt_read == 0 && !fail_on_eof) return (0); /* EOF */ @@ -261,11 +275,14 @@ read_bytes(FILE *fp, void *buf, size_t bytes_to_read, int fail_on_eof, static int read_block(FILE *fp, pcap_t *p, struct block_cursor *cursor, char *errbuf) { + struct pcap_ng_sf *ps; int status; struct block_header bhdr; u_char *bdata; size_t data_remaining; + ps = p->priv; + status = read_bytes(fp, &bhdr, sizeof(bhdr), 0, errbuf); if (status <= 0) return (status); /* error or EOF */ @@ -284,7 +301,7 @@ read_block(FILE *fp, pcap_t *p, struct block_cursor *cursor, char *errbuf) */ if (bhdr.total_length > 16*1024*1024) { pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "pcap-ng block size %u > maximum %u", + "pcapng block size %u > maximum %u", bhdr.total_length, 16*1024*1024); return (-1); } @@ -296,7 +313,7 @@ read_block(FILE *fp, pcap_t *p, struct block_cursor *cursor, char *errbuf) if (bhdr.total_length < sizeof(struct block_header) + sizeof(struct block_trailer)) { pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "block in pcap-ng dump file has a length of %u < %lu", + "block in pcapng dump file has a length of %u < %lu", bhdr.total_length, (unsigned long)(sizeof(struct block_header) + sizeof(struct block_trailer))); return (-1); @@ -307,10 +324,15 @@ read_block(FILE *fp, pcap_t *p, struct block_cursor *cursor, char *errbuf) */ if (p->bufsize < bhdr.total_length) { /* - * No - make it big enough. + * No - make it big enough, unless it's too big. */ void *bigger_buffer; + if (bhdr.total_length > ps->max_blocksize) { + pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, "block is larger than maximum block size %u", + ps->max_blocksize); + return (-1); + } bigger_buffer = realloc(p->buffer, bhdr.total_length); if (bigger_buffer == NULL) { pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, "out of memory"); @@ -350,7 +372,7 @@ get_from_block_data(struct block_cursor *cursor, size_t chunk_size, */ if (cursor->data_remaining < chunk_size) { pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "block of type %u in pcap-ng dump file is too short", + "block of type %u in pcapng dump file is too short", cursor->block_type); return (NULL); } @@ -412,7 +434,7 @@ get_optvalue_from_block_data(struct block_cursor *cursor, static int process_idb_options(pcap_t *p, struct block_cursor *cursor, u_int *tsresol, - u_int64_t *tsoffset, int *is_binary, char *errbuf) + uint64_t *tsoffset, int *is_binary, char *errbuf) { struct option_header *opthdr; void *optvalue; @@ -535,7 +557,7 @@ add_interface(pcap_t *p, struct block_cursor *cursor, char *errbuf) { struct pcap_ng_sf *ps; u_int tsresol; - u_int64_t tsoffset; + uint64_t tsoffset; int is_binary; ps = p->priv; @@ -569,7 +591,7 @@ add_interface(pcap_t *p, struct block_cursor *cursor, char *errbuf) * However, it doesn't complain that one of the * multiplications below could overflow, which is * a real, albeit extremely unlikely, problem (you'd - * need a pcap-ng file with tens of millions of + * need a pcapng file with tens of millions of * interfaces).) */ new_ifaces_size = 1; @@ -701,7 +723,7 @@ add_interface(pcap_t *p, struct block_cursor *cursor, char *errbuf) } /* - * Check whether this is a pcap-ng savefile and, if it is, extract the + * Check whether this is a pcapng savefile and, if it is, extract the * relevant information from the header. */ pcap_t * @@ -727,7 +749,7 @@ pcap_ng_check_header(bpf_u_int32 magic, FILE *fp, u_int precision, char *errbuf, /* * Check whether the first 4 bytes of the file are the block - * type for a pcap-ng savefile. + * type for a pcapng savefile. */ if (magic != BT_SHB) { /* @@ -736,7 +758,7 @@ pcap_ng_check_header(bpf_u_int32 magic, FILE *fp, u_int precision, char *errbuf, * UN*X and DOS/Windows text file format and, if it * does, look for the byte-order magic number in * the appropriate place and, if we find it, report - * this as possibly being a pcap-ng file transferred + * this as possibly being a pcapng file transferred * between UN*X and Windows in text file format? */ return (NULL); /* nope */ @@ -756,32 +778,30 @@ pcap_ng_check_header(bpf_u_int32 magic, FILE *fp, u_int precision, char *errbuf, amt_read = fread(&total_length, 1, sizeof(total_length), fp); if (amt_read < sizeof(total_length)) { if (ferror(fp)) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "error reading dump file: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "error reading dump file"); *err = 1; return (NULL); /* fail */ } /* * Possibly a weird short text file, so just say - * "not pcap-ng". + * "not pcapng". */ return (NULL); } amt_read = fread(&byte_order_magic, 1, sizeof(byte_order_magic), fp); if (amt_read < sizeof(byte_order_magic)) { if (ferror(fp)) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "error reading dump file: %s", - pcap_strerror(errno)); + pcap_fmt_errmsg_for_errno(errbuf, PCAP_ERRBUF_SIZE, + errno, "error reading dump file"); *err = 1; return (NULL); /* fail */ } /* * Possibly a weird short text file, so just say - * "not pcap-ng". + * "not pcapng". */ return (NULL); } @@ -789,7 +809,7 @@ pcap_ng_check_header(bpf_u_int32 magic, FILE *fp, u_int precision, char *errbuf, byte_order_magic = SWAPLONG(byte_order_magic); if (byte_order_magic != BYTE_ORDER_MAGIC) { /* - * Not a pcap-ng file. + * Not a pcapng file. */ return (NULL); } @@ -802,7 +822,7 @@ pcap_ng_check_header(bpf_u_int32 magic, FILE *fp, u_int precision, char *errbuf, */ if (total_length < sizeof(*bhdrp) + sizeof(*shbp) + sizeof(struct block_trailer)) { pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "Section Header Block in pcap-ng dump file has a length of %u < %lu", + "Section Header Block in pcapng dump file has a length of %u < %lu", total_length, (unsigned long)(sizeof(*bhdrp) + sizeof(*shbp) + sizeof(struct block_trailer))); *err = 1; @@ -810,7 +830,7 @@ pcap_ng_check_header(bpf_u_int32 magic, FILE *fp, u_int precision, char *errbuf, } /* - * OK, this is a good pcap-ng file. + * OK, this is a good pcapng file. * Allocate a pcap_t for it. */ p = pcap_open_offline_common(errbuf, sizeof (struct pcap_ng_sf)); @@ -855,7 +875,11 @@ pcap_ng_check_header(bpf_u_int32 magic, FILE *fp, u_int precision, char *errbuf, * Packet Block containing a full-size Ethernet frame, and * leaving room for some options. * - * If we find a bigger block, we reallocate the buffer. + * If we find a bigger block, we reallocate the buffer, up to + * the maximum size. We start out with a maximum size based + * on a maximum snapshot length of MAXIMUM_SNAPLEN; if we see + * any link-layer header types with a larger maximum snapshot + * length, we boost the maximum. */ p->bufsize = 2048; if (p->bufsize < total_length) @@ -867,6 +891,7 @@ pcap_ng_check_header(bpf_u_int32 magic, FILE *fp, u_int precision, char *errbuf, *err = 1; return (NULL); } + ps->max_blocksize = MAX_BLOCKSIZE(MAXIMUM_SNAPLEN); /* * Copy the stuff we've read to the buffer, and read the rest @@ -898,7 +923,7 @@ pcap_ng_check_header(bpf_u_int32 magic, FILE *fp, u_int precision, char *errbuf, if (! (shbp->major_version == PCAP_NG_VERSION_MAJOR && shbp->minor_version == PCAP_NG_VERSION_MINOR)) { pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "unsupported pcap-ng savefile version %u.%u", + "unsupported pcapng savefile version %u.%u", shbp->major_version, shbp->minor_version); goto fail; } @@ -947,17 +972,6 @@ pcap_ng_check_header(bpf_u_int32 magic, FILE *fp, u_int precision, char *errbuf, } /* - * Interface capture length sanity check - */ - if (idbp->snaplen > MAXIMUM_SNAPLEN) { - pcap_snprintf(errbuf, PCAP_ERRBUF_SIZE, - "invalid interface capture length %u, " - "bigger than maximum of %u", - idbp->snaplen, MAXIMUM_SNAPLEN); - goto fail; - } - - /* * Try to add this interface. */ if (!add_interface(p, &cursor, errbuf)) @@ -988,9 +1002,28 @@ pcap_ng_check_header(bpf_u_int32 magic, FILE *fp, u_int precision, char *errbuf, done: p->tzoff = 0; /* XXX - not used in pcap */ p->snapshot = idbp->snaplen; + if (p->snapshot <= 0) { + /* + * Bogus snapshot length; use the maximum for this + * link-layer type as a fallback. + * + * XXX - the only reason why snapshot is signed is + * that pcap_snapshot() returns an int, not an + * unsigned int. + */ + p->snapshot = max_snaplen_for_dlt(idbp->linktype); + } p->linktype = linktype_to_dlt(idbp->linktype); p->linktype_ext = 0; + /* + * If the maximum block size for a packet with the maximum + * snapshot length for this DLT_ is bigger than the current + * maximum block size, increase the maximum. + */ + if (MAX_BLOCKSIZE(max_snaplen_for_dlt(p->linktype)) > ps->max_blocksize) + ps->max_blocksize = MAX_BLOCKSIZE(max_snaplen_for_dlt(p->linktype)); + p->next_packet_op = pcap_ng_next_packet; p->cleanup_op = pcap_ng_cleanup; @@ -1031,7 +1064,7 @@ pcap_ng_next_packet(pcap_t *p, struct pcap_pkthdr *hdr, u_char **data) struct interface_description_block *idbp; struct section_header_block *shbp; FILE *fp = p->rfile; - u_int64_t t, sec, frac; + uint64_t t, sec, frac; /* * Look for an Enhanced Packet Block, a Simple Packet Block, @@ -1067,13 +1100,13 @@ pcap_ng_next_packet(pcap_t *p, struct pcap_pkthdr *hdr, u_char **data) interface_id = SWAPLONG(epbp->interface_id); hdr->caplen = SWAPLONG(epbp->caplen); hdr->len = SWAPLONG(epbp->len); - t = ((u_int64_t)SWAPLONG(epbp->timestamp_high)) << 32 | + t = ((uint64_t)SWAPLONG(epbp->timestamp_high)) << 32 | SWAPLONG(epbp->timestamp_low); } else { interface_id = epbp->interface_id; hdr->caplen = epbp->caplen; hdr->len = epbp->len; - t = ((u_int64_t)epbp->timestamp_high) << 32 | + t = ((uint64_t)epbp->timestamp_high) << 32 | epbp->timestamp_low; } goto found; @@ -1132,13 +1165,13 @@ pcap_ng_next_packet(pcap_t *p, struct pcap_pkthdr *hdr, u_char **data) interface_id = SWAPSHORT(pbp->interface_id); hdr->caplen = SWAPLONG(pbp->caplen); hdr->len = SWAPLONG(pbp->len); - t = ((u_int64_t)SWAPLONG(pbp->timestamp_high)) << 32 | + t = ((uint64_t)SWAPLONG(pbp->timestamp_high)) << 32 | SWAPLONG(pbp->timestamp_low); } else { interface_id = pbp->interface_id; hdr->caplen = pbp->caplen; hdr->len = pbp->len; - t = ((u_int64_t)pbp->timestamp_high) << 32 | + t = ((uint64_t)pbp->timestamp_high) << 32 | pbp->timestamp_low; } goto found; @@ -1247,7 +1280,7 @@ pcap_ng_next_packet(pcap_t *p, struct pcap_pkthdr *hdr, u_char **data) */ if (shbp->major_version != PCAP_NG_VERSION_MAJOR) { pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, - "unknown pcap-ng savefile major version number %u", + "unknown pcapng savefile major version number %u", shbp->major_version); return (-1); } @@ -1286,6 +1319,13 @@ found: return (-1); } + if (hdr->caplen > (bpf_u_int32)p->snapshot) { + pcap_snprintf(p->errbuf, PCAP_ERRBUF_SIZE, + "invalid packet capture length %u, bigger than " + "snaplen of %d", hdr->caplen, p->snapshot); + return (-1); + } + /* * Convert the time stamp to seconds and fractions of a second, * with the fractions being in units of the file-supplied resolution. diff --git a/freebsd/contrib/libpcap/sf-pcap-ng.h b/freebsd/contrib/libpcap/sf-pcapng.h index 3c93498f..d99b0d4e 100644 --- a/freebsd/contrib/libpcap/sf-pcap-ng.h +++ b/freebsd/contrib/libpcap/sf-pcapng.h @@ -18,13 +18,13 @@ * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. * - * sf-pcap-ng.h - pcap-ng-file-format-specific routines + * sf-pcapng.h - pcapng-file-format-specific routines * - * Used to read pcap-ng savefiles. + * Used to read pcapng savefiles. */ -#ifndef sf_pcap_ng_h -#define sf_pcap_ng_h +#ifndef sf_pcapng_h +#define sf_pcapng_h extern pcap_t *pcap_ng_check_header(bpf_u_int32 magic, FILE *fp, u_int precision, char *errbuf, int *err); diff --git a/freebsd/contrib/libpcap/varattrs.h b/freebsd/contrib/libpcap/varattrs.h new file mode 100644 index 00000000..05bfe8cd --- /dev/null +++ b/freebsd/contrib/libpcap/varattrs.h @@ -0,0 +1,59 @@ +/* -*- Mode: c; tab-width: 8; indent-tabs-mode: 1; c-basic-offset: 8; -*- */ +/* + * Copyright (c) 1993, 1994, 1995, 1996, 1997 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the Computer Systems + * Engineering Group at Lawrence Berkeley Laboratory. + * 4. Neither the name of the University nor of the Laboratory may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef varattrs_h +#define varattrs_h + +#include <pcap/compiler-tests.h> + +/* + * Attributes to apply to variables, using various compiler-specific + * extensions. + */ + +#if __has_attribute(unused) \ + || PCAP_IS_AT_LEAST_GNUC_VERSION(2,0) + /* + * Compiler with support for __attribute__((unused)), or GCC 2.0 and + * later, so it supports __attribute__((unused)). + */ + #define _U_ __attribute__((unused)) +#else + /* + * We don't know of any way to mark a variable as unused. + */ + #define _U_ +#endif + +#endif diff --git a/freebsd/contrib/libxo/libxo/libxo.c b/freebsd/contrib/libxo/libxo/libxo.c index 4577adb2..2a78aea1 100644 --- a/freebsd/contrib/libxo/libxo/libxo.c +++ b/freebsd/contrib/libxo/libxo/libxo.c @@ -4182,6 +4182,59 @@ xo_format_title (xo_handle_t *xop, xo_field_info_t *xfip, } } +/* + * strspn() with a string length + */ +static ssize_t +xo_strnspn (const char *str, size_t len, const char *accept) +{ + ssize_t i; + const char *cp, *ep; + + for (i = 0, cp = str, ep = str + len; cp < ep && *cp != '\0'; i++, cp++) { + if (strchr(accept, *cp) == NULL) + break; + } + + return i; +} + +/* + * Decide if a format string should be considered "numeric", + * in the sense that the number does not need to be quoted. + * This means that it consists only of a single numeric field + * with nothing exotic or "interesting". This means that + * static values are never considered numeric. + */ +static int +xo_format_is_numeric (const char *fmt, ssize_t flen) +{ + if (flen <= 0 || *fmt++ != '%') /* Must start with '%' */ + return FALSE; + flen -= 1; + + /* Handle leading flags; don't want "#" since JSON can't handle hex */ + ssize_t spn = xo_strnspn(fmt, flen, "0123456789.*+ -"); + if (spn >= flen) + return FALSE; + + fmt += spn; /* Move along the input string */ + flen -= spn; + + /* Handle the length modifiers */ + spn = xo_strnspn(fmt, flen, "hljtqz"); + if (spn >= flen) + return FALSE; + + fmt += spn; /* Move along the input string */ + flen -= spn; + + if (flen != 1) /* Should only be one character left */ + return FALSE; + + return (strchr("diouDOUeEfFgG", *fmt) == NULL) ? FALSE : TRUE; +} + static void xo_format_prep (xo_handle_t *xop, xo_xff_flags_t flags) { @@ -4410,10 +4463,10 @@ xo_format_value (xo_handle_t *xop, const char *name, ssize_t nlen, quote = 0; fmt = "true"; /* JSON encodes empty tags as a boolean true */ flen = 4; - } else if (strchr("diouDOUeEfFgG", fmt[flen - 1]) == NULL) - quote = 1; - else + } else if (xo_format_is_numeric(fmt, flen)) quote = 0; + else + quote = 1; if (nlen == 0) { static char missing[] = "missing-field-name"; diff --git a/freebsd/contrib/tcpdump/tcpdump.c b/freebsd/contrib/tcpdump/tcpdump.c index 26761ad7..75896da3 100644 --- a/freebsd/contrib/tcpdump/tcpdump.c +++ b/freebsd/contrib/tcpdump/tcpdump.c @@ -89,7 +89,6 @@ The Regents of the University of California. All rights reserved.\n"; */ #ifdef HAVE_CAPSICUM #include <sys/capsicum.h> -#include <sys/sysctl.h> #include <sys/nv.h> #include <sys/ioccom.h> #include <net/bpf.h> @@ -126,6 +125,10 @@ The Regents of the University of California. All rights reserved.\n"; #endif /* HAVE_CAP_NG_H */ #endif /* HAVE_LIBCAP_NG */ +#ifdef __FreeBSD__ +#include <sys/sysctl.h> +#endif /* __FreeBSD__ */ + #include "netdissect.h" #include "interface.h" #include "addrtoname.h" @@ -1129,6 +1132,10 @@ open_interface(const char *device, netdissect_options *ndo, char *ebuf) sysctlbyname(sysctl, parent, &s, NULL, 0); strlcpy(newdev, device, sizeof(newdev)); /* Suggest a new wlan device. */ + /* FIXME: incrementing the index this way is not going to work well + * when the index is 9 or greater but the only consequence in this + * specific case would be an error message that looks a bit odd. + */ newdev[strlen(newdev)-1]++; error("%s is not a monitor mode VAP\n" "To create a new monitor mode VAP use:\n" diff --git a/freebsd/lib/libc/db/mpool/mpool-compat.c b/freebsd/lib/libc/db/mpool/mpool-compat.c index 37bbdd79..44b06285 100644 --- a/freebsd/lib/libc/db/mpool/mpool-compat.c +++ b/freebsd/lib/libc/db/mpool/mpool-compat.c @@ -4,7 +4,6 @@ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009 Xin LI <delphij@FreeBSD.org> - * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions diff --git a/freebsd/lib/libc/net/getnameinfo.c b/freebsd/lib/libc/net/getnameinfo.c index d553ee5c..219e149d 100644 --- a/freebsd/lib/libc/net/getnameinfo.c +++ b/freebsd/lib/libc/net/getnameinfo.c @@ -483,7 +483,6 @@ getnameinfo_link(const struct afd *afd, * IFT_OTHER (netinet/ip_ipip.c) */ /* default below is believed correct for all these. */ - case IFT_ARCNET: case IFT_ETHER: case IFT_FDDI: case IFT_HIPPI: diff --git a/freebsd/lib/libmemstat/memstat.c b/freebsd/lib/libmemstat/memstat.c index 60da9b00..d264829e 100644 --- a/freebsd/lib/libmemstat/memstat.c +++ b/freebsd/lib/libmemstat/memstat.c @@ -31,6 +31,7 @@ */ #include <sys/param.h> +#include <sys/queue.h> #include <sys/sysctl.h> #include <err.h> diff --git a/freebsd/sbin/dhclient/dhclient.c b/freebsd/sbin/dhclient/dhclient.c index d9bae19c..e0af9431 100644 --- a/freebsd/sbin/dhclient/dhclient.c +++ b/freebsd/sbin/dhclient/dhclient.c @@ -851,11 +851,23 @@ bind_lease(struct interface_info *ip) opt = &ip->client->new->options[DHO_INTERFACE_MTU]; if (opt->len == sizeof(u_int16_t)) { - u_int16_t mtu = be16dec(opt->data); - if (mtu < MIN_MTU) - warning("mtu size %u < %d: ignored", (unsigned)mtu, MIN_MTU); + u_int16_t mtu = 0; + bool supersede = (ip->client->config->default_actions[DHO_INTERFACE_MTU] == + ACTION_SUPERSEDE); + + if (supersede) + mtu = getUShort(ip->client->config->defaults[DHO_INTERFACE_MTU].data); else + mtu = be16dec(opt->data); + + if (mtu < MIN_MTU) { + /* Treat 0 like a user intentionally doesn't want to change MTU and, + * therefore, warning is not needed */ + if (!supersede || mtu != 0) + warning("mtu size %u < %d: ignored", (unsigned)mtu, MIN_MTU); + } else { interface_set_mtu_unpriv(privfd, mtu); + } } /* Write out the new lease. */ diff --git a/freebsd/sbin/ifconfig/ifclone.c b/freebsd/sbin/ifconfig/ifclone.c index 749e12c0..ee1f7b22 100644 --- a/freebsd/sbin/ifconfig/ifclone.c +++ b/freebsd/sbin/ifconfig/ifclone.c @@ -202,7 +202,7 @@ static void clone_Copt_cb(const char *optarg __unused) { list_cloners(); - exit(0); + exit(exit_code); } static struct option clone_Copt = { .opt = "C", .opt_usage = "[-C]", .cb = clone_Copt_cb }; diff --git a/freebsd/sbin/ifconfig/ifconfig.c b/freebsd/sbin/ifconfig/ifconfig.c index e625e42b..56b4f6d7 100644 --- a/freebsd/sbin/ifconfig/ifconfig.c +++ b/freebsd/sbin/ifconfig/ifconfig.c @@ -117,6 +117,7 @@ int printifname = 0; int supmedia = 0; int printkeys = 0; /* Print keying material for interfaces. */ +int exit_code = 0; /* Formatter Strings */ char *f_inet, *f_inet6, *f_ether, *f_addr; @@ -566,7 +567,7 @@ main(int argc, char *argv[]) errx(1, "%s: cloning name too long", ifname); ifconfig(argc, argv, 1, NULL); - exit(0); + exit(exit_code); } #ifdef JAIL /* @@ -580,7 +581,7 @@ main(int argc, char *argv[]) errx(1, "%s: interface name too long", ifname); ifconfig(argc, argv, 0, NULL); - exit(0); + exit(exit_code); } #endif errx(1, "interface %s does not exist", ifname); @@ -678,7 +679,7 @@ main(int argc, char *argv[]) freeifaddrs(ifap); freeformat(); - exit(0); + exit(exit_code); } static struct afswtch *afs = NULL; @@ -1573,8 +1574,8 @@ static struct cmd basic_cmds[] = { DEF_CMD("-wol_magic", -IFCAP_WOL_MAGIC, setifcap), DEF_CMD("txrtlmt", IFCAP_TXRTLMT, setifcap), DEF_CMD("-txrtlmt", -IFCAP_TXRTLMT, setifcap), - DEF_CMD("hwrxtsmp", IFCAP_HWRXTSTMP, setifcap), - DEF_CMD("-hwrxtsmp", -IFCAP_HWRXTSTMP, setifcap), + DEF_CMD("hwrxtstmp", IFCAP_HWRXTSTMP, setifcap), + DEF_CMD("-hwrxtstmp", -IFCAP_HWRXTSTMP, setifcap), DEF_CMD("normal", -IFF_LINK0, setifflags), DEF_CMD("compress", IFF_LINK0, setifflags), DEF_CMD("noicmp", IFF_LINK1, setifflags), diff --git a/freebsd/sbin/ifconfig/ifconfig.h b/freebsd/sbin/ifconfig/ifconfig.h index 97fe20e1..0e2fcba2 100644 --- a/freebsd/sbin/ifconfig/ifconfig.h +++ b/freebsd/sbin/ifconfig/ifconfig.h @@ -136,6 +136,7 @@ extern int printkeys; extern int newaddr; extern int verbose; extern int printifname; +extern int exit_code; void setifcap(const char *, int value, int s, const struct afswtch *); diff --git a/freebsd/sbin/ifconfig/ifgroup.c b/freebsd/sbin/ifconfig/ifgroup.c index ae8e2b8d..a487f060 100644 --- a/freebsd/sbin/ifconfig/ifgroup.c +++ b/freebsd/sbin/ifconfig/ifgroup.c @@ -162,7 +162,7 @@ printgroup(const char *groupname) if (ioctl(s, SIOCGIFGMEMB, (caddr_t)&ifgr) == -1) { if (errno == EINVAL || errno == ENOTTY || errno == ENOENT) - exit(0); + exit(exit_code); else err(1, "SIOCGIFGMEMB"); } @@ -181,7 +181,7 @@ printgroup(const char *groupname) } free(ifgr.ifgr_groups); - exit(0); + exit(exit_code); } static struct cmd group_cmds[] = { diff --git a/freebsd/sbin/ifconfig/iflagg.c b/freebsd/sbin/ifconfig/iflagg.c index d659ed57..4952196a 100644 --- a/freebsd/sbin/ifconfig/iflagg.c +++ b/freebsd/sbin/ifconfig/iflagg.c @@ -53,9 +53,17 @@ setlaggport(const char *val, int d, int s, const struct afswtch *afp) strlcpy(rp.rp_ifname, name, sizeof(rp.rp_ifname)); strlcpy(rp.rp_portname, val, sizeof(rp.rp_portname)); - /* Don't choke if the port is already in this lagg. */ - if (ioctl(s, SIOCSLAGGPORT, &rp) && errno != EEXIST) - err(1, "SIOCSLAGGPORT"); + /* + * Do not exit with an error here. Doing so permits a + * failed NIC to take down an entire lagg. + * + * Don't error at all if the port is already in the lagg. + */ + if (ioctl(s, SIOCSLAGGPORT, &rp) && errno != EEXIST) { + warnx("%s %s: SIOCSLAGGPORT: %s", + name, val, strerror(errno)); + exit_code = 1; + } } static void diff --git a/freebsd/sbin/ifconfig/ifmedia.c b/freebsd/sbin/ifconfig/ifmedia.c index b986d548..ac2af8b6 100644 --- a/freebsd/sbin/ifconfig/ifmedia.c +++ b/freebsd/sbin/ifconfig/ifmedia.c @@ -195,14 +195,6 @@ media_status(int s) printf("no carrier"); break; - case IFM_FDDI: - case IFM_TOKEN: - if (ifmr.ifm_status & IFM_ACTIVE) - printf("inserted"); - else - printf("no ring"); - break; - case IFM_IEEE80211: if (ifmr.ifm_status & IFM_ACTIVE) { /* NB: only sta mode associates */ @@ -430,24 +422,6 @@ static struct ifmedia_description ifm_subtype_ethernet_aliases[] = static struct ifmedia_description ifm_subtype_ethernet_option_descriptions[] = IFM_SUBTYPE_ETHERNET_OPTION_DESCRIPTIONS; -static struct ifmedia_description ifm_subtype_tokenring_descriptions[] = - IFM_SUBTYPE_TOKENRING_DESCRIPTIONS; - -static struct ifmedia_description ifm_subtype_tokenring_aliases[] = - IFM_SUBTYPE_TOKENRING_ALIASES; - -static struct ifmedia_description ifm_subtype_tokenring_option_descriptions[] = - IFM_SUBTYPE_TOKENRING_OPTION_DESCRIPTIONS; - -static struct ifmedia_description ifm_subtype_fddi_descriptions[] = - IFM_SUBTYPE_FDDI_DESCRIPTIONS; - -static struct ifmedia_description ifm_subtype_fddi_aliases[] = - IFM_SUBTYPE_FDDI_ALIASES; - -static struct ifmedia_description ifm_subtype_fddi_option_descriptions[] = - IFM_SUBTYPE_FDDI_OPTION_DESCRIPTIONS; - static struct ifmedia_description ifm_subtype_ieee80211_descriptions[] = IFM_SUBTYPE_IEEE80211_DESCRIPTIONS; @@ -525,42 +499,6 @@ static struct ifmedia_type_to_subtype ifmedia_types_to_subtypes[] = { { { &ifm_subtype_shared_descriptions[0], 0 }, { &ifm_subtype_shared_aliases[0], 1 }, - { &ifm_subtype_tokenring_descriptions[0], 0 }, - { &ifm_subtype_tokenring_aliases[0], 1 }, - { NULL, 0 }, - }, - { - { &ifm_shared_option_descriptions[0], 0 }, - { &ifm_shared_option_aliases[0], 1 }, - { &ifm_subtype_tokenring_option_descriptions[0], 0 }, - { NULL, 0 }, - }, - { - { NULL, 0 }, - }, - }, - { - { - { &ifm_subtype_shared_descriptions[0], 0 }, - { &ifm_subtype_shared_aliases[0], 1 }, - { &ifm_subtype_fddi_descriptions[0], 0 }, - { &ifm_subtype_fddi_aliases[0], 1 }, - { NULL, 0 }, - }, - { - { &ifm_shared_option_descriptions[0], 0 }, - { &ifm_shared_option_aliases[0], 1 }, - { &ifm_subtype_fddi_option_descriptions[0], 0 }, - { NULL, 0 }, - }, - { - { NULL, 0 }, - }, - }, - { - { - { &ifm_subtype_shared_descriptions[0], 0 }, - { &ifm_subtype_shared_aliases[0], 1 }, { &ifm_subtype_ieee80211_descriptions[0], 0 }, { &ifm_subtype_ieee80211_aliases[0], 1 }, { NULL, 0 }, diff --git a/freebsd/sbin/ifconfig/rtems-bsd-ifconfig-data.h b/freebsd/sbin/ifconfig/rtems-bsd-ifconfig-data.h index 3b1956ea..a605646d 100644 --- a/freebsd/sbin/ifconfig/rtems-bsd-ifconfig-data.h +++ b/freebsd/sbin/ifconfig/rtems-bsd-ifconfig-data.h @@ -15,6 +15,7 @@ RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, extern char *f_inet6); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, extern char name[]); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, extern int clearaddr); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, extern int doalias); +RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, extern int exit_code); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, extern int newaddr); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, extern int noload); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, extern int printifname); diff --git a/freebsd/sbin/ifconfig/rtems-bsd-ifconfig-ifmedia-data.h b/freebsd/sbin/ifconfig/rtems-bsd-ifconfig-ifmedia-data.h index 84d16c92..911589ac 100644 --- a/freebsd/sbin/ifconfig/rtems-bsd-ifconfig-ifmedia-data.h +++ b/freebsd/sbin/ifconfig/rtems-bsd-ifconfig-ifmedia-data.h @@ -13,17 +13,11 @@ RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, static struct ifmedia_description RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, static struct ifmedia_description ifm_subtype_ethernet_aliases[]); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, static struct ifmedia_description ifm_subtype_ethernet_descriptions[]); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, static struct ifmedia_description ifm_subtype_ethernet_option_descriptions[]); -RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, static struct ifmedia_description ifm_subtype_fddi_aliases[]); -RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, static struct ifmedia_description ifm_subtype_fddi_descriptions[]); -RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, static struct ifmedia_description ifm_subtype_fddi_option_descriptions[]); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, static struct ifmedia_description ifm_subtype_ieee80211_aliases[]); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, static struct ifmedia_description ifm_subtype_ieee80211_descriptions[]); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, static struct ifmedia_description ifm_subtype_ieee80211_option_descriptions[]); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, static struct ifmedia_description ifm_subtype_shared_aliases[]); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, static struct ifmedia_description ifm_subtype_shared_descriptions[]); -RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, static struct ifmedia_description ifm_subtype_tokenring_aliases[]); -RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, static struct ifmedia_description ifm_subtype_tokenring_descriptions[]); -RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, static struct ifmedia_description ifm_subtype_tokenring_option_descriptions[]); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, static struct ifmedia_description ifm_type_descriptions[]); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, static struct ifmedia_type_to_subtype ifmedia_types_to_subtypes[]); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_ifconfig, static struct ifmediareq *ifmedia_getstate_ifmr); diff --git a/freebsd/sbin/ifconfig/rtems-bsd-ifconfig-namespace.h b/freebsd/sbin/ifconfig/rtems-bsd-ifconfig-namespace.h index 03cd73d6..e7488dfe 100644 --- a/freebsd/sbin/ifconfig/rtems-bsd-ifconfig-namespace.h +++ b/freebsd/sbin/ifconfig/rtems-bsd-ifconfig-namespace.h @@ -23,6 +23,7 @@ #define descr _bsd_ifconfig_descr #define descrlen _bsd_ifconfig_descrlen #define doalias _bsd_ifconfig_doalias +#define exit_code _bsd_ifconfig_exit_code #define f_addr _bsd_ifconfig_f_addr #define f_ether _bsd_ifconfig_f_ether #define f_inet _bsd_ifconfig_f_inet diff --git a/freebsd/sbin/pfctl/pfctl.c b/freebsd/sbin/pfctl/pfctl.c index c814a816..ca7d6f38 100644 --- a/freebsd/sbin/pfctl/pfctl.c +++ b/freebsd/sbin/pfctl/pfctl.c @@ -85,6 +85,9 @@ void usage(void); int pfctl_enable(int, int); int pfctl_disable(int, int); int pfctl_clear_stats(int, int); +int pfctl_get_skip_ifaces(void); +int pfctl_check_skip_ifaces(char *); +int pfctl_clear_skip_ifaces(struct pfctl *); int pfctl_clear_interface_flags(int, int); int pfctl_clear_rules(int, int, char *); int pfctl_clear_nat(int, int, char *); @@ -124,6 +127,7 @@ const char *pfctl_lookup_option(char *, const char * const *); static struct pf_anchor_global pf_anchors; static struct pf_anchor pf_main_anchor; +static struct pfr_buffer skip_b; static const char *clearopt; static char *rulesopt; @@ -317,6 +321,44 @@ pfctl_clear_stats(int dev, int opts) } int +pfctl_get_skip_ifaces(void) +{ + bzero(&skip_b, sizeof(skip_b)); + skip_b.pfrb_type = PFRB_IFACES; + for (;;) { + pfr_buf_grow(&skip_b, skip_b.pfrb_size); + skip_b.pfrb_size = skip_b.pfrb_msize; + if (pfi_get_ifaces(NULL, skip_b.pfrb_caddr, &skip_b.pfrb_size)) + err(1, "pfi_get_ifaces"); + if (skip_b.pfrb_size <= skip_b.pfrb_msize) + break; + } + return (0); +} + +int +pfctl_check_skip_ifaces(char *ifname) +{ + struct pfi_kif *p; + + PFRB_FOREACH(p, &skip_b) + if ((p->pfik_flags & PFI_IFLAG_SKIP) && !strcmp(ifname, p->pfik_name)) + p->pfik_flags &= ~PFI_IFLAG_SKIP; + return (0); +} + +int +pfctl_clear_skip_ifaces(struct pfctl *pf) +{ + struct pfi_kif *p; + + PFRB_FOREACH(p, &skip_b) + if (p->pfik_flags & PFI_IFLAG_SKIP) + pfctl_set_interface_flags(pf, p->pfik_name, PFI_IFLAG_SKIP, 0); + return (0); +} + +int pfctl_clear_interface_flags(int dev, int opts) { struct pfioc_iface pi; @@ -1505,6 +1547,8 @@ pfctl_rules(int dev, char *filename, int opts, int optimize, else goto _error; } + if (loadopt & PFCTL_FLAG_OPTION) + pfctl_clear_skip_ifaces(&pf); if ((pf.loadopt & PFCTL_FLAG_FILTER && (pfctl_load_ruleset(&pf, path, rs, PF_RULESET_SCRUB, 0))) || @@ -1915,6 +1959,7 @@ pfctl_set_interface_flags(struct pfctl *pf, char *ifname, int flags, int how) } else { if (ioctl(pf->dev, DIOCSETIFFLAG, &pi)) err(1, "DIOCSETIFFLAG"); + pfctl_check_skip_ifaces(ifname); } } return (0); @@ -2405,7 +2450,7 @@ main(int argc, char *argv[]) if ((rulesopt != NULL) && (loadopt & PFCTL_FLAG_OPTION) && !anchorname[0]) - if (pfctl_clear_interface_flags(dev, opts | PF_OPT_QUIET)) + if (pfctl_get_skip_ifaces()) error = 1; if (rulesopt != NULL && !(opts & (PF_OPT_MERGE|PF_OPT_NOACTION)) && diff --git a/freebsd/sbin/pfctl/rtems-bsd-pfctl-namespace.h b/freebsd/sbin/pfctl/rtems-bsd-pfctl-namespace.h index 8aba20df..46053fd9 100644 --- a/freebsd/sbin/pfctl/rtems-bsd-pfctl-namespace.h +++ b/freebsd/sbin/pfctl/rtems-bsd-pfctl-namespace.h @@ -79,11 +79,13 @@ #define pfctl_add_pool _bsd_pfctl_pfctl_add_pool #define pfctl_add_rule _bsd_pfctl_pfctl_add_rule #define pfctl_addrprefix _bsd_pfctl_pfctl_addrprefix +#define pfctl_check_skip_ifaces _bsd_pfctl_pfctl_check_skip_ifaces #define pfctl_clear_altq _bsd_pfctl_pfctl_clear_altq #define pfctl_clear_interface_flags _bsd_pfctl_pfctl_clear_interface_flags #define pfctl_clear_nat _bsd_pfctl_pfctl_clear_nat #define pfctl_clear_pool _bsd_pfctl_pfctl_clear_pool #define pfctl_clear_rules _bsd_pfctl_pfctl_clear_rules +#define pfctl_clear_skip_ifaces _bsd_pfctl_pfctl_clear_skip_ifaces #define pfctl_clear_src_nodes _bsd_pfctl_pfctl_clear_src_nodes #define pfctl_clear_states _bsd_pfctl_pfctl_clear_states #define pfctl_clear_stats _bsd_pfctl_pfctl_clear_stats @@ -92,6 +94,7 @@ #define pfctl_enable _bsd_pfctl_pfctl_enable #define pfctl_fopen _bsd_pfctl_pfctl_fopen #define pfctl_get_pool _bsd_pfctl_pfctl_get_pool +#define pfctl_get_skip_ifaces _bsd_pfctl_pfctl_get_skip_ifaces #define pfctl_id_kill_states _bsd_pfctl_pfctl_id_kill_states #define pfctl_init_options _bsd_pfctl_pfctl_init_options #define pfctl_kill_src_nodes _bsd_pfctl_pfctl_kill_src_nodes diff --git a/freebsd/sbin/pfctl/rtems-bsd-pfctl-pfctl-data.h b/freebsd/sbin/pfctl/rtems-bsd-pfctl-pfctl-data.h index 614cb6b2..3fba09b1 100644 --- a/freebsd/sbin/pfctl/rtems-bsd-pfctl-pfctl-data.h +++ b/freebsd/sbin/pfctl/rtems-bsd-pfctl-pfctl-data.h @@ -20,3 +20,4 @@ RTEMS_LINKER_RWSET_CONTENT(bsd_prog_pfctl, static int src_node_killers); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_pfctl, static int state_killers); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_pfctl, static struct pf_anchor pf_main_anchor); RTEMS_LINKER_RWSET_CONTENT(bsd_prog_pfctl, static struct pf_anchor_global pf_anchors); +RTEMS_LINKER_RWSET_CONTENT(bsd_prog_pfctl, static struct pfr_buffer skip_b); diff --git a/freebsd/sys/arm/at91/at91_mci.c b/freebsd/sys/arm/at91/at91_mci.c index 6f5ace83..c25983b7 100644 --- a/freebsd/sys/arm/at91/at91_mci.c +++ b/freebsd/sys/arm/at91/at91_mci.c @@ -4,7 +4,7 @@ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2006 Bernd Walter. All rights reserved. - * Copyright (c) 2006 M. Warner Losh. All rights reserved. + * Copyright (c) 2006 M. Warner Losh. * Copyright (c) 2010 Greg Ansley. All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/freebsd/sys/arm/at91/at91_mcireg.h b/freebsd/sys/arm/at91/at91_mcireg.h index eec67857..80acf48d 100644 --- a/freebsd/sys/arm/at91/at91_mcireg.h +++ b/freebsd/sys/arm/at91/at91_mcireg.h @@ -2,7 +2,7 @@ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2006 Berndt Walter. All rights reserved. - * Copyright (c) 2006 M. Warner Losh. All rights reserved. + * Copyright (c) 2006 M. Warner Losh. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions diff --git a/freebsd/sys/arm/at91/at91_pdcreg.h b/freebsd/sys/arm/at91/at91_pdcreg.h index 88389e00..659804bd 100644 --- a/freebsd/sys/arm/at91/at91_pdcreg.h +++ b/freebsd/sys/arm/at91/at91_pdcreg.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * - * Copyright (c) 2006 M. Warner Losh. All rights reserved. + * Copyright (c) 2006 M. Warner Losh. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions diff --git a/freebsd/sys/arm/ti/am335x/am335x_prcm.c b/freebsd/sys/arm/ti/am335x/am335x_prcm.c index 83faafbd..eb27555e 100644 --- a/freebsd/sys/arm/ti/am335x/am335x_prcm.c +++ b/freebsd/sys/arm/ti/am335x/am335x_prcm.c @@ -138,6 +138,7 @@ struct am335x_prcm_softc { struct resource * res[2]; bus_space_tag_t bst; bus_space_handle_t bsh; + int attach_done; }; static struct resource_spec am335x_prcm_spec[] = { @@ -426,7 +427,6 @@ static int am335x_prcm_attach(device_t dev) { struct am335x_prcm_softc *sc = device_get_softc(dev); - unsigned int sysclk, fclk; if (am335x_prcm_sc) return (ENXIO); @@ -444,6 +444,24 @@ am335x_prcm_attach(device_t dev) ti_cpu_reset = am335x_prcm_reset; #endif /* __rtems__ */ + return (0); +} + +static void +am335x_prcm_new_pass(device_t dev) +{ + struct am335x_prcm_softc *sc = device_get_softc(dev); + unsigned int sysclk, fclk; + + sc = device_get_softc(dev); + if (sc->attach_done || + bus_current_pass < (BUS_PASS_TIMER + BUS_PASS_ORDER_EARLY)) { + bus_generic_new_pass(dev); + return; + } + + sc->attach_done = 1; + if (am335x_clk_get_sysclk_freq(NULL, &sysclk) != 0) sysclk = 0; if (am335x_clk_get_arm_fclk_freq(NULL, &fclk) != 0) @@ -451,15 +469,24 @@ am335x_prcm_attach(device_t dev) if (sysclk && fclk) device_printf(dev, "Clocks: System %u.%01u MHz, CPU %u MHz\n", sysclk/1000000, (sysclk % 1000000)/100000, fclk/1000000); - else + else { device_printf(dev, "can't read frequencies yet (SCM device not ready?)\n"); + goto fail; + } - return (0); + return; + +fail: + device_detach(dev); + return; } static device_method_t am335x_prcm_methods[] = { DEVMETHOD(device_probe, am335x_prcm_probe), DEVMETHOD(device_attach, am335x_prcm_attach), + + /* Bus interface */ + DEVMETHOD(bus_new_pass, am335x_prcm_new_pass), { 0, 0 } }; @@ -472,7 +499,7 @@ static driver_t am335x_prcm_driver = { static devclass_t am335x_prcm_devclass; EARLY_DRIVER_MODULE(am335x_prcm, simplebus, am335x_prcm_driver, - am335x_prcm_devclass, 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_EARLY); + am335x_prcm_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE); MODULE_VERSION(am335x_prcm, 1); MODULE_DEPEND(am335x_prcm, ti_scm, 1, 1, 1); diff --git a/freebsd/sys/arm/ti/ti_hwmods.c b/freebsd/sys/arm/ti/ti_hwmods.c index 0e703472..450679a7 100644 --- a/freebsd/sys/arm/ti/ti_hwmods.c +++ b/freebsd/sys/arm/ti/ti_hwmods.c @@ -112,7 +112,7 @@ ti_hwmods_get_clock(device_t dev) if ((node = ofw_bus_get_node(dev)) == 0) return (INVALID_CLK_IDENT); - if ((len = OF_getprop_alloc(node, "ti,hwmods", 1, (void**)&name)) <= 0) + if ((len = OF_getprop_alloc(node, "ti,hwmods", (void**)&name)) <= 0) return (INVALID_CLK_IDENT); buf = name; @@ -150,7 +150,7 @@ int ti_hwmods_contains(device_t dev, const char *hwmod) if ((node = ofw_bus_get_node(dev)) == 0) return (0); - if ((len = OF_getprop_alloc(node, "ti,hwmods", 1, (void**)&name)) <= 0) + if ((len = OF_getprop_alloc(node, "ti,hwmods", (void**)&name)) <= 0) return (0); buf = name; @@ -184,7 +184,7 @@ ti_hwmods_get_unit(device_t dev, const char *hwmod) if ((node = ofw_bus_get_node(dev)) == 0) return (0); - if ((len = OF_getprop_alloc(node, "ti,hwmods", 1, (void**)&name)) <= 0) + if ((len = OF_getprop_alloc(node, "ti,hwmods", (void**)&name)) <= 0) return (0); buf = name; diff --git a/freebsd/sys/cam/nvme/nvme_all.h b/freebsd/sys/cam/nvme/nvme_all.h index fa229846..e31c1e5e 100644 --- a/freebsd/sys/cam/nvme/nvme_all.h +++ b/freebsd/sys/cam/nvme/nvme_all.h @@ -2,7 +2,6 @@ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2015 Netflix, Inc - * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions diff --git a/freebsd/sys/contrib/ck/include/ck_backoff.h b/freebsd/sys/contrib/ck/include/ck_backoff.h new file mode 100644 index 00000000..82a4f215 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/ck_backoff.h @@ -0,0 +1,57 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_BACKOFF_H +#define CK_BACKOFF_H + +#include <ck_cc.h> +#include <ck_pr.h> + +#ifndef CK_BACKOFF_CEILING +#define CK_BACKOFF_CEILING ((1 << 20) - 1) +#endif + +#define CK_BACKOFF_INITIALIZER (1 << 9) + +typedef unsigned int ck_backoff_t; + +/* + * This is a exponential back-off implementation. + */ +CK_CC_INLINE static void +ck_backoff_eb(unsigned int *c) +{ + unsigned int i, ceiling; + + ceiling = *c; + for (i = 0; i < ceiling; i++) + ck_pr_barrier(); + + *c = ceiling <<= ceiling < CK_BACKOFF_CEILING; + return; +} + +#endif /* CK_BACKOFF_H */ diff --git a/freebsd/sys/contrib/ck/include/ck_cc.h b/freebsd/sys/contrib/ck/include/ck_cc.h new file mode 100644 index 00000000..9a152a3c --- /dev/null +++ b/freebsd/sys/contrib/ck/include/ck_cc.h @@ -0,0 +1,173 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * Copyright 2014 Paul Khuong. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_CC_H +#define CK_CC_H + +#if defined(__GNUC__) || defined(__SUNPRO_C) +#include "gcc/ck_cc.h" +#endif + +#ifndef CK_CC_RESTRICT +#define CK_CC_RESTRICT +#endif + +#ifndef CK_CC_INLINE +#define CK_CC_INLINE inline +#endif + +#ifndef CK_CC_FORCE_INLINE +#define CK_CC_FORCE_INLINE inline +#endif + +#define CK_CC_DECONST_PTR(X) ((void *)(uintptr_t)(X)) + +/* + * Container function. + * This relies on (compiler) implementation-defined behavior. + */ +#define CK_CC_CONTAINER(F, T, M, N) \ + CK_CC_INLINE static T * \ + N(F *p) \ + { \ + F *n = p; \ + return (T *)(void *)(((char *)n) - ((size_t)&((T *)0)->M)); \ + } + +#define CK_CC_PAD(x) union { char pad[x]; } + +#ifndef CK_CC_ALIASED +#define CK_CC_ALIASED +#endif + +#ifndef CK_CC_UNUSED +#define CK_CC_UNUSED +#endif + +#ifndef CK_CC_USED +#define CK_CC_USED +#endif + +#ifndef CK_CC_IMM +#define CK_CC_IMM +#endif + +#ifndef CK_CC_PACKED +#define CK_CC_PACKED +#endif + +#ifndef CK_CC_WEAKREF +#define CK_CC_WEAKREF +#endif + +#ifndef CK_CC_ALIGN +#define CK_CC_ALIGN(X) +#endif + +#ifndef CK_CC_CACHELINE +#define CK_CC_CACHELINE +#endif + +#ifndef CK_CC_LIKELY +#define CK_CC_LIKELY(x) x +#endif + +#ifndef CK_CC_UNLIKELY +#define CK_CC_UNLIKELY(x) x +#endif + +#ifndef CK_CC_TYPEOF +#define CK_CC_TYPEOF(X, DEFAULT) (DEFAULT) +#endif + +#define CK_F_CC_FFS_G(L, T) \ +CK_CC_INLINE static int \ +ck_cc_##L(T v) \ +{ \ + unsigned int i; \ + \ + if (v == 0) \ + return 0; \ + \ + for (i = 1; (v & 1) == 0; i++, v >>= 1); \ + return i; \ +} + +#ifndef CK_F_CC_FFS +#define CK_F_CC_FFS +CK_F_CC_FFS_G(ffs, unsigned int) +#endif /* CK_F_CC_FFS */ + +#ifndef CK_F_CC_FFSL +#define CK_F_CC_FFSL +CK_F_CC_FFS_G(ffsl, unsigned long) +#endif /* CK_F_CC_FFSL */ + +#ifndef CK_F_CC_FFSLL +#define CK_F_CC_FFSLL +CK_F_CC_FFS_G(ffsll, unsigned long long) +#endif /* CK_F_CC_FFSLL */ + +#undef CK_F_CC_FFS_G + +#ifndef CK_F_CC_CTZ +#define CK_F_CC_CTZ +CK_CC_INLINE static int +ck_cc_ctz(unsigned int x) +{ + unsigned int i; + + if (x == 0) + return 0; + + for (i = 0; (x & 1) == 0; i++, x >>= 1); + return i; +} +#endif + +#ifndef CK_F_CC_POPCOUNT +#define CK_F_CC_POPCOUNT +CK_CC_INLINE static int +ck_cc_popcount(unsigned int x) +{ + unsigned int acc; + + for (acc = 0; x != 0; x >>= 1) + acc += x & 1; + + return acc; +} +#endif + + +#ifdef __cplusplus +#define CK_CPP_CAST(type, arg) static_cast<type>(arg) +#else +#define CK_CPP_CAST(type, arg) arg +#endif + +#endif /* CK_CC_H */ diff --git a/freebsd/sys/contrib/ck/include/ck_epoch.h b/freebsd/sys/contrib/ck/include/ck_epoch.h new file mode 100644 index 00000000..58f3d28a --- /dev/null +++ b/freebsd/sys/contrib/ck/include/ck_epoch.h @@ -0,0 +1,281 @@ +/* + * Copyright 2011-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_EPOCH_H +#define CK_EPOCH_H + +/* + * The implementation here is inspired from the work described in: + * Fraser, K. 2004. Practical Lock-Freedom. PhD Thesis, University + * of Cambridge Computing Laboratory. + */ + +#include <ck_cc.h> +#include <ck_md.h> +#include <ck_pr.h> +#include <ck_stack.h> +#include <ck_stdbool.h> + +#ifndef CK_EPOCH_LENGTH +#define CK_EPOCH_LENGTH 4 +#endif + +/* + * This is used for sense detection with-respect to concurrent + * epoch sections. + */ +#define CK_EPOCH_SENSE (2) + +struct ck_epoch_entry; +typedef struct ck_epoch_entry ck_epoch_entry_t; +typedef void ck_epoch_cb_t(ck_epoch_entry_t *); + +/* + * This should be embedded into objects you wish to be the target of + * ck_epoch_cb_t functions (with ck_epoch_call). + */ +struct ck_epoch_entry { + ck_epoch_cb_t *function; + ck_stack_entry_t stack_entry; +}; + +/* + * A section object may be passed to every begin-end pair to allow for + * forward progress guarantees with-in prolonged active sections. + */ +struct ck_epoch_section { + unsigned int bucket; +}; +typedef struct ck_epoch_section ck_epoch_section_t; + +/* + * Return pointer to ck_epoch_entry container object. + */ +#define CK_EPOCH_CONTAINER(T, M, N) \ + CK_CC_CONTAINER(struct ck_epoch_entry, T, M, N) + +struct ck_epoch_ref { + unsigned int epoch; + unsigned int count; +}; + +struct ck_epoch_record { + ck_stack_entry_t record_next; + struct ck_epoch *global; + unsigned int state; + unsigned int epoch; + unsigned int active; + struct { + struct ck_epoch_ref bucket[CK_EPOCH_SENSE]; + } local CK_CC_CACHELINE; + unsigned int n_pending; + unsigned int n_peak; + unsigned int n_dispatch; + void *ct; + ck_stack_t pending[CK_EPOCH_LENGTH]; +} CK_CC_CACHELINE; +typedef struct ck_epoch_record ck_epoch_record_t; + +struct ck_epoch { + unsigned int epoch; + unsigned int n_free; + ck_stack_t records; +}; +typedef struct ck_epoch ck_epoch_t; + +/* + * Internal functions. + */ +void _ck_epoch_addref(ck_epoch_record_t *, ck_epoch_section_t *); +bool _ck_epoch_delref(ck_epoch_record_t *, ck_epoch_section_t *); + +CK_CC_FORCE_INLINE static void * +ck_epoch_record_ct(const ck_epoch_record_t *record) +{ + + return ck_pr_load_ptr(&record->ct); +} + +/* + * Marks the beginning of an epoch-protected section. + */ +CK_CC_FORCE_INLINE static void +ck_epoch_begin(ck_epoch_record_t *record, ck_epoch_section_t *section) +{ + struct ck_epoch *epoch = record->global; + + /* + * Only observe new epoch if thread is not recursing into a read + * section. + */ + if (record->active == 0) { + unsigned int g_epoch; + + /* + * It is possible for loads to be re-ordered before the store + * is committed into the caller's epoch and active fields. + * For this reason, store to load serialization is necessary. + */ +#if defined(CK_MD_TSO) + ck_pr_fas_uint(&record->active, 1); + ck_pr_fence_atomic_load(); +#else + ck_pr_store_uint(&record->active, 1); + ck_pr_fence_memory(); +#endif + + /* + * This load is allowed to be re-ordered prior to setting + * active flag due to monotonic nature of the global epoch. + * However, stale values lead to measurable performance + * degradation in some torture tests so we disallow early load + * of global epoch. + */ + g_epoch = ck_pr_load_uint(&epoch->epoch); + ck_pr_store_uint(&record->epoch, g_epoch); + } else { + ck_pr_store_uint(&record->active, record->active + 1); + } + + if (section != NULL) + _ck_epoch_addref(record, section); + + return; +} + +/* + * Marks the end of an epoch-protected section. Returns true if no more + * sections exist for the caller. + */ +CK_CC_FORCE_INLINE static bool +ck_epoch_end(ck_epoch_record_t *record, ck_epoch_section_t *section) +{ + + ck_pr_fence_release(); + ck_pr_store_uint(&record->active, record->active - 1); + + if (section != NULL) + return _ck_epoch_delref(record, section); + + return record->active == 0; +} + +/* + * Defers the execution of the function pointed to by the "cb" + * argument until an epoch counter loop. This allows for a + * non-blocking deferral. + * + * We can get away without a fence here due to the monotonic nature + * of the epoch counter. Worst case, this will result in some delays + * before object destruction. + */ +CK_CC_FORCE_INLINE static void +ck_epoch_call(ck_epoch_record_t *record, + ck_epoch_entry_t *entry, + ck_epoch_cb_t *function) +{ + struct ck_epoch *epoch = record->global; + unsigned int e = ck_pr_load_uint(&epoch->epoch); + unsigned int offset = e & (CK_EPOCH_LENGTH - 1); + + record->n_pending++; + entry->function = function; + ck_stack_push_spnc(&record->pending[offset], &entry->stack_entry); + return; +} + +/* + * Same as ck_epoch_call, but allows for records to be shared and is reentrant. + */ +CK_CC_FORCE_INLINE static void +ck_epoch_call_strict(ck_epoch_record_t *record, + ck_epoch_entry_t *entry, + ck_epoch_cb_t *function) +{ + struct ck_epoch *epoch = record->global; + unsigned int e = ck_pr_load_uint(&epoch->epoch); + unsigned int offset = e & (CK_EPOCH_LENGTH - 1); + + ck_pr_inc_uint(&record->n_pending); + entry->function = function; + + /* Store fence is implied by push operation. */ + ck_stack_push_upmc(&record->pending[offset], &entry->stack_entry); + return; +} + +/* + * This callback is used for synchronize_wait to allow for custom blocking + * behavior. + */ +typedef void ck_epoch_wait_cb_t(ck_epoch_t *, ck_epoch_record_t *, + void *); + +/* + * Return latest epoch value. This operation provides load ordering. + */ +CK_CC_FORCE_INLINE static unsigned int +ck_epoch_value(const ck_epoch_t *ep) +{ + + ck_pr_fence_load(); + return ck_pr_load_uint(&ep->epoch); +} + +void ck_epoch_init(ck_epoch_t *); + +/* + * Attempts to recycle an unused epoch record. If one is successfully + * allocated, the record context pointer is also updated. + */ +ck_epoch_record_t *ck_epoch_recycle(ck_epoch_t *, void *); + +/* + * Registers an epoch record. An optional context pointer may be passed that + * is retrievable with ck_epoch_record_ct. + */ +void ck_epoch_register(ck_epoch_t *, ck_epoch_record_t *, void *); + +/* + * Marks a record as available for re-use by a subsequent recycle operation. + * Note that the record cannot be physically destroyed. + */ +void ck_epoch_unregister(ck_epoch_record_t *); + +bool ck_epoch_poll(ck_epoch_record_t *); +bool ck_epoch_poll_deferred(struct ck_epoch_record *record, ck_stack_t *deferred); +void ck_epoch_synchronize(ck_epoch_record_t *); +void ck_epoch_synchronize_wait(ck_epoch_t *, ck_epoch_wait_cb_t *, void *); +void ck_epoch_barrier(ck_epoch_record_t *); +void ck_epoch_barrier_wait(ck_epoch_record_t *, ck_epoch_wait_cb_t *, void *); + +/* + * Reclaim entries associated with a record. This is safe to call only on + * the caller's record or records that are using call_strict. + */ +void ck_epoch_reclaim(ck_epoch_record_t *); + +#endif /* CK_EPOCH_H */ diff --git a/freebsd/sys/contrib/ck/include/ck_limits.h b/freebsd/sys/contrib/ck/include/ck_limits.h new file mode 100644 index 00000000..c8749550 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/ck_limits.h @@ -0,0 +1,48 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__linux__) && defined(__KERNEL__) +#include <linux/kernel.h> + +#ifndef UINT8_MAX +#define UINT8_MAX ((u8)(~0U)) +#endif +#ifndef UINT16_MAX +#define UINT16_MAX USHRT_MAX +#endif +#ifndef UINT32_MAX +#define UINT32_MAX UINT_MAX +#endif +#ifndef UINT64_MAX +#define UINT64_MAX ULLONG_MAX +#endif + +#elif defined(__FreeBSD__) && defined(_KERNEL) +#include <sys/stdint.h> +#include <sys/limits.h> +#else +#include <limits.h> +#endif /* __linux__ && __KERNEL__ */ diff --git a/freebsd/sys/contrib/ck/include/ck_md.h b/freebsd/sys/contrib/ck/include/ck_md.h new file mode 100644 index 00000000..3a69584e --- /dev/null +++ b/freebsd/sys/contrib/ck/include/ck_md.h @@ -0,0 +1,136 @@ +/* + * Copyright 2018 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD: head/sys/contrib/ck/include/ck_md.h 329388 2018-02-16 17:50:06Z cog +net $ + */ + +/* + * This header file is meant for use of Concurrency Kit in the FreeBSD kernel. + */ + +#ifndef CK_MD_H +#define CK_MD_H + +#include <sys/param.h> + +#ifndef __rtems__ +#ifndef _KERNEL +#error This header file is meant for the FreeBSD kernel. +#endif /* _KERNEL */ +#endif /* __rtems__ */ + +#ifndef CK_MD_CACHELINE +/* + * FreeBSD's CACHE_LINE macro is a compile-time maximum cache-line size for an + * architecture, defined to be 128 bytes by default on x86*. Even in presence + * of adjacent sector prefetch, this doesn't make sense from a modeling + * perspective. + */ +#if defined(__amd64__) || defined(__i386__) +#define CK_MD_CACHELINE (64) +#else +#define CK_MD_CACHELINE (CACHE_LINE_SIZE) +#endif /* !__amd64__ && !__i386__ */ +#endif /* CK_MD_CACHELINE */ + +#ifndef CK_MD_PAGESIZE +#define CK_MD_PAGESIZE (PAGE_SIZE) +#endif + +/* + * Once FreeBSD has a mechanism to detect RTM, this can be enabled and RTM + * facilities can be called. These facilities refer to TSX. + */ +#ifndef CK_MD_RTM_DISABLE +#define CK_MD_RTM_DISABLE +#endif /* CK_MD_RTM_DISABLE */ + +/* + * Do not enable pointer-packing-related (VMA) optimizations in kernel-space. + */ +#ifndef CK_MD_POINTER_PACK_DISABLE +#define CK_MD_POINTER_PACK_DISABLE +#endif /* CK_MD_POINTER_PACK_DISABLE */ + +/* + * The following would be used for pointer-packing tricks, disabled for the + * kernel. + */ +#ifndef CK_MD_VMA_BITS_UNKNOWN +#define CK_MD_VMA_BITS_UNKNOWN +#endif /* CK_MD_VMA_BITS_UNKNOWN */ + +/* + * Do not enable double operations in kernel-space. + */ +#ifndef CK_PR_DISABLE_DOUBLE +#define CK_PR_DISABLE_DOUBLE +#endif /* CK_PR_DISABLE_DOUBLE */ + +/* + * If building for a uni-processor target, then enable the uniprocessor + * feature flag. This, among other things, will remove the lock prefix. + */ +#ifndef SMP +#define CK_MD_UMP +#endif /* SMP */ + +/* + * Disable the use of compiler builtin functions. + */ +#define CK_MD_CC_BUILTIN_DISABLE 1 + +/* + * CK expects those, which are normally defined by the build system. + */ +#if defined(__i386__) && !defined(__x86__) +#define __x86__ +/* + * If x86 becomes more relevant, we may want to consider importing in + * __mbk() to avoid potential issues around false sharing. + */ +#define CK_MD_TSO +#define CK_MD_SSE_DISABLE 1 +#elif defined(__amd64__) +#define CK_MD_TSO +#elif defined(__sparc64__) && !defined(__sparcv9__) +#define __sparcv9__ +#define CK_MD_TSO +#elif defined(__powerpc64__) && !defined(__ppc64__) +#define __ppc64__ +#elif defined(__powerpc__) && !defined(__ppc__) +#define __ppc__ +#endif + +/* If no memory model has been defined, assume RMO. */ +#if !defined(CK_MD_RMO) && !defined(CK_MD_TSO) && !defined(CK_MD_PSO) +#define CK_MD_RMO +#endif + +#define CK_VERSION "0.7.0" +#define CK_GIT_SHA "db5db44" + +#endif /* CK_MD_H */ diff --git a/freebsd/sys/contrib/ck/include/ck_pr.h b/freebsd/sys/contrib/ck/include/ck_pr.h new file mode 100644 index 00000000..7fa57a8e --- /dev/null +++ b/freebsd/sys/contrib/ck/include/ck_pr.h @@ -0,0 +1,1225 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * Copyright 2011 David Joseph. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_H +#define CK_PR_H + +#include <ck_cc.h> +#include <ck_limits.h> +#include <ck_md.h> +#include <ck_stdint.h> +#include <ck_stdbool.h> + +#ifndef CK_USE_CC_BUILTINS +#if defined(__x86_64__) +#include "gcc/x86_64/ck_pr.h" +#elif defined(__x86__) +#include "gcc/x86/ck_pr.h" +#elif defined(__sparcv9__) +#include "gcc/sparcv9/ck_pr.h" +#elif defined(__ppc64__) +#include "gcc/ppc64/ck_pr.h" +#elif defined(__s390x__) +#include "gcc/s390x/ck_pr.h" +#elif defined(__ppc__) +#include "gcc/ppc/ck_pr.h" +#elif defined(__arm__) +#if __ARM_ARCH >= 6 +#include "gcc/arm/ck_pr.h" +#else +#include "gcc/arm/ck_pr_armv4.h" +#endif +#elif defined(__aarch64__) +#include "gcc/aarch64/ck_pr.h" +#elif !defined(__GNUC__) +#error Your platform is unsupported +#endif +#endif /* !CK_USE_CC_BUILTINS */ + +#if defined(__GNUC__) +#include "gcc/ck_pr.h" +#endif + +#define CK_PR_FENCE_EMIT(T) \ + CK_CC_INLINE static void \ + ck_pr_fence_##T(void) \ + { \ + ck_pr_fence_strict_##T(); \ + return; \ + } +#define CK_PR_FENCE_NOOP(T) \ + CK_CC_INLINE static void \ + ck_pr_fence_##T(void) \ + { \ + ck_pr_barrier(); \ + return; \ + } + +/* + * None of the currently supported platforms allow for data-dependent + * load ordering. + */ +CK_PR_FENCE_NOOP(load_depends) +#define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends + +/* + * In memory models where atomic operations do not have serializing + * effects, atomic read-modify-write operations are modeled as stores. + */ +#if defined(CK_MD_RMO) +/* + * Only stores to the same location have a global + * ordering. + */ +CK_PR_FENCE_EMIT(atomic) +CK_PR_FENCE_EMIT(atomic_load) +CK_PR_FENCE_EMIT(atomic_store) +CK_PR_FENCE_EMIT(store_atomic) +CK_PR_FENCE_EMIT(load_atomic) +CK_PR_FENCE_EMIT(load_store) +CK_PR_FENCE_EMIT(store_load) +CK_PR_FENCE_EMIT(load) +CK_PR_FENCE_EMIT(store) +CK_PR_FENCE_EMIT(memory) +CK_PR_FENCE_EMIT(acquire) +CK_PR_FENCE_EMIT(release) +CK_PR_FENCE_EMIT(acqrel) +CK_PR_FENCE_EMIT(lock) +CK_PR_FENCE_EMIT(unlock) +#elif defined(CK_MD_PSO) +/* + * Anything can be re-ordered with respect to stores. + * Otherwise, loads are executed in-order. + */ +CK_PR_FENCE_EMIT(atomic) +CK_PR_FENCE_NOOP(atomic_load) +CK_PR_FENCE_EMIT(atomic_store) +CK_PR_FENCE_EMIT(store_atomic) +CK_PR_FENCE_NOOP(load_atomic) +CK_PR_FENCE_EMIT(load_store) +CK_PR_FENCE_EMIT(store_load) +CK_PR_FENCE_NOOP(load) +CK_PR_FENCE_EMIT(store) +CK_PR_FENCE_EMIT(memory) +CK_PR_FENCE_EMIT(acquire) +CK_PR_FENCE_EMIT(release) +CK_PR_FENCE_EMIT(acqrel) +CK_PR_FENCE_EMIT(lock) +CK_PR_FENCE_EMIT(unlock) +#elif defined(CK_MD_TSO) +/* + * Only loads are re-ordered and only with respect to + * prior stores. Atomic operations are serializing. + */ +CK_PR_FENCE_NOOP(atomic) +CK_PR_FENCE_NOOP(atomic_load) +CK_PR_FENCE_NOOP(atomic_store) +CK_PR_FENCE_NOOP(store_atomic) +CK_PR_FENCE_NOOP(load_atomic) +CK_PR_FENCE_NOOP(load_store) +CK_PR_FENCE_EMIT(store_load) +CK_PR_FENCE_NOOP(load) +CK_PR_FENCE_NOOP(store) +CK_PR_FENCE_EMIT(memory) +CK_PR_FENCE_NOOP(acquire) +CK_PR_FENCE_NOOP(release) +CK_PR_FENCE_NOOP(acqrel) +CK_PR_FENCE_NOOP(lock) +CK_PR_FENCE_NOOP(unlock) +#else +#error "No memory model has been defined." +#endif /* CK_MD_TSO */ + +#undef CK_PR_FENCE_EMIT +#undef CK_PR_FENCE_NOOP + +#ifndef CK_F_PR_RFO +#define CK_F_PR_RFO +CK_CC_INLINE static void +ck_pr_rfo(const void *m) +{ + + (void)m; + return; +} +#endif /* CK_F_PR_RFO */ + +#define CK_PR_STORE_SAFE(DST, VAL, TYPE) \ + ck_pr_md_store_##TYPE( \ + ((void)sizeof(*(DST) = (VAL)), (DST)), \ + (VAL)) + +#define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr) +#define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char) +#ifndef CK_PR_DISABLE_DOUBLE +#define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double) +#endif +#define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint) +#define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int) +#define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32) +#define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16) +#define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8) + +#define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL)) + +#ifdef CK_F_PR_LOAD_64 +#define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64) +#endif /* CK_F_PR_LOAD_64 */ + +#define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC)) +#define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC)) + +#define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC)) +#define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char) +#ifndef CK_PR_DISABLE_DOUBLE +#define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double) +#endif +#define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint) +#define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int) +#define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32) +#define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16) +#define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8) + +#ifdef CK_F_PR_LOAD_64 +#define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64) +#endif /* CK_F_PR_LOAD_64 */ + +#define CK_PR_BIN(K, S, M, T, P, C) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target, T value) \ + { \ + T previous; \ + C punt; \ + punt = ck_pr_md_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(previous P value), \ + &previous) == false) \ + ck_pr_stall(); \ + \ + return; \ + } + +#define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T) + +#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) + +#ifndef CK_F_PR_ADD_CHAR +#define CK_F_PR_ADD_CHAR +CK_PR_BIN_S(add, char, char, +) +#endif /* CK_F_PR_ADD_CHAR */ + +#ifndef CK_F_PR_SUB_CHAR +#define CK_F_PR_SUB_CHAR +CK_PR_BIN_S(sub, char, char, -) +#endif /* CK_F_PR_SUB_CHAR */ + +#ifndef CK_F_PR_AND_CHAR +#define CK_F_PR_AND_CHAR +CK_PR_BIN_S(and, char, char, &) +#endif /* CK_F_PR_AND_CHAR */ + +#ifndef CK_F_PR_XOR_CHAR +#define CK_F_PR_XOR_CHAR +CK_PR_BIN_S(xor, char, char, ^) +#endif /* CK_F_PR_XOR_CHAR */ + +#ifndef CK_F_PR_OR_CHAR +#define CK_F_PR_OR_CHAR +CK_PR_BIN_S(or, char, char, |) +#endif /* CK_F_PR_OR_CHAR */ + +#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ + +#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) + +#ifndef CK_F_PR_ADD_INT +#define CK_F_PR_ADD_INT +CK_PR_BIN_S(add, int, int, +) +#endif /* CK_F_PR_ADD_INT */ + +#ifndef CK_F_PR_SUB_INT +#define CK_F_PR_SUB_INT +CK_PR_BIN_S(sub, int, int, -) +#endif /* CK_F_PR_SUB_INT */ + +#ifndef CK_F_PR_AND_INT +#define CK_F_PR_AND_INT +CK_PR_BIN_S(and, int, int, &) +#endif /* CK_F_PR_AND_INT */ + +#ifndef CK_F_PR_XOR_INT +#define CK_F_PR_XOR_INT +CK_PR_BIN_S(xor, int, int, ^) +#endif /* CK_F_PR_XOR_INT */ + +#ifndef CK_F_PR_OR_INT +#define CK_F_PR_OR_INT +CK_PR_BIN_S(or, int, int, |) +#endif /* CK_F_PR_OR_INT */ + +#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ + +#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ + !defined(CK_PR_DISABLE_DOUBLE) + +#ifndef CK_F_PR_ADD_DOUBLE +#define CK_F_PR_ADD_DOUBLE +CK_PR_BIN_S(add, double, double, +) +#endif /* CK_F_PR_ADD_DOUBLE */ + +#ifndef CK_F_PR_SUB_DOUBLE +#define CK_F_PR_SUB_DOUBLE +CK_PR_BIN_S(sub, double, double, -) +#endif /* CK_F_PR_SUB_DOUBLE */ + +#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ + +#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) + +#ifndef CK_F_PR_ADD_UINT +#define CK_F_PR_ADD_UINT +CK_PR_BIN_S(add, uint, unsigned int, +) +#endif /* CK_F_PR_ADD_UINT */ + +#ifndef CK_F_PR_SUB_UINT +#define CK_F_PR_SUB_UINT +CK_PR_BIN_S(sub, uint, unsigned int, -) +#endif /* CK_F_PR_SUB_UINT */ + +#ifndef CK_F_PR_AND_UINT +#define CK_F_PR_AND_UINT +CK_PR_BIN_S(and, uint, unsigned int, &) +#endif /* CK_F_PR_AND_UINT */ + +#ifndef CK_F_PR_XOR_UINT +#define CK_F_PR_XOR_UINT +CK_PR_BIN_S(xor, uint, unsigned int, ^) +#endif /* CK_F_PR_XOR_UINT */ + +#ifndef CK_F_PR_OR_UINT +#define CK_F_PR_OR_UINT +CK_PR_BIN_S(or, uint, unsigned int, |) +#endif /* CK_F_PR_OR_UINT */ + +#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ + +#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) + +#ifndef CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_PTR +CK_PR_BIN(add, ptr, void, uintptr_t, +, void *) +#endif /* CK_F_PR_ADD_PTR */ + +#ifndef CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_PTR +CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *) +#endif /* CK_F_PR_SUB_PTR */ + +#ifndef CK_F_PR_AND_PTR +#define CK_F_PR_AND_PTR +CK_PR_BIN(and, ptr, void, uintptr_t, &, void *) +#endif /* CK_F_PR_AND_PTR */ + +#ifndef CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_PTR +CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *) +#endif /* CK_F_PR_XOR_PTR */ + +#ifndef CK_F_PR_OR_PTR +#define CK_F_PR_OR_PTR +CK_PR_BIN(or, ptr, void, uintptr_t, |, void *) +#endif /* CK_F_PR_OR_PTR */ + +#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ + +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) + +#ifndef CK_F_PR_ADD_64 +#define CK_F_PR_ADD_64 +CK_PR_BIN_S(add, 64, uint64_t, +) +#endif /* CK_F_PR_ADD_64 */ + +#ifndef CK_F_PR_SUB_64 +#define CK_F_PR_SUB_64 +CK_PR_BIN_S(sub, 64, uint64_t, -) +#endif /* CK_F_PR_SUB_64 */ + +#ifndef CK_F_PR_AND_64 +#define CK_F_PR_AND_64 +CK_PR_BIN_S(and, 64, uint64_t, &) +#endif /* CK_F_PR_AND_64 */ + +#ifndef CK_F_PR_XOR_64 +#define CK_F_PR_XOR_64 +CK_PR_BIN_S(xor, 64, uint64_t, ^) +#endif /* CK_F_PR_XOR_64 */ + +#ifndef CK_F_PR_OR_64 +#define CK_F_PR_OR_64 +CK_PR_BIN_S(or, 64, uint64_t, |) +#endif /* CK_F_PR_OR_64 */ + +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ + +#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) + +#ifndef CK_F_PR_ADD_32 +#define CK_F_PR_ADD_32 +CK_PR_BIN_S(add, 32, uint32_t, +) +#endif /* CK_F_PR_ADD_32 */ + +#ifndef CK_F_PR_SUB_32 +#define CK_F_PR_SUB_32 +CK_PR_BIN_S(sub, 32, uint32_t, -) +#endif /* CK_F_PR_SUB_32 */ + +#ifndef CK_F_PR_AND_32 +#define CK_F_PR_AND_32 +CK_PR_BIN_S(and, 32, uint32_t, &) +#endif /* CK_F_PR_AND_32 */ + +#ifndef CK_F_PR_XOR_32 +#define CK_F_PR_XOR_32 +CK_PR_BIN_S(xor, 32, uint32_t, ^) +#endif /* CK_F_PR_XOR_32 */ + +#ifndef CK_F_PR_OR_32 +#define CK_F_PR_OR_32 +CK_PR_BIN_S(or, 32, uint32_t, |) +#endif /* CK_F_PR_OR_32 */ + +#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ + +#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) + +#ifndef CK_F_PR_ADD_16 +#define CK_F_PR_ADD_16 +CK_PR_BIN_S(add, 16, uint16_t, +) +#endif /* CK_F_PR_ADD_16 */ + +#ifndef CK_F_PR_SUB_16 +#define CK_F_PR_SUB_16 +CK_PR_BIN_S(sub, 16, uint16_t, -) +#endif /* CK_F_PR_SUB_16 */ + +#ifndef CK_F_PR_AND_16 +#define CK_F_PR_AND_16 +CK_PR_BIN_S(and, 16, uint16_t, &) +#endif /* CK_F_PR_AND_16 */ + +#ifndef CK_F_PR_XOR_16 +#define CK_F_PR_XOR_16 +CK_PR_BIN_S(xor, 16, uint16_t, ^) +#endif /* CK_F_PR_XOR_16 */ + +#ifndef CK_F_PR_OR_16 +#define CK_F_PR_OR_16 +CK_PR_BIN_S(or, 16, uint16_t, |) +#endif /* CK_F_PR_OR_16 */ + +#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ + +#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) + +#ifndef CK_F_PR_ADD_8 +#define CK_F_PR_ADD_8 +CK_PR_BIN_S(add, 8, uint8_t, +) +#endif /* CK_F_PR_ADD_8 */ + +#ifndef CK_F_PR_SUB_8 +#define CK_F_PR_SUB_8 +CK_PR_BIN_S(sub, 8, uint8_t, -) +#endif /* CK_F_PR_SUB_8 */ + +#ifndef CK_F_PR_AND_8 +#define CK_F_PR_AND_8 +CK_PR_BIN_S(and, 8, uint8_t, &) +#endif /* CK_F_PR_AND_8 */ + +#ifndef CK_F_PR_XOR_8 +#define CK_F_PR_XOR_8 +CK_PR_BIN_S(xor, 8, uint8_t, ^) +#endif /* CK_F_PR_XOR_8 */ + +#ifndef CK_F_PR_OR_8 +#define CK_F_PR_OR_8 +CK_PR_BIN_S(or, 8, uint8_t, |) +#endif /* CK_F_PR_OR_8 */ + +#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ + +#undef CK_PR_BIN_S +#undef CK_PR_BIN + +#define CK_PR_BTX(K, S, M, T, P, C, R) \ + CK_CC_INLINE static bool \ + ck_pr_##K##_##S(M *target, unsigned int offset) \ + { \ + T previous; \ + C punt; \ + punt = ck_pr_md_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, (C)previous, \ + (C)(previous P (R ((T)1 << offset))), &previous) == false) \ + ck_pr_stall(); \ + return ((previous >> offset) & 1); \ + } + +#define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R) + +#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) + +#ifndef CK_F_PR_BTC_INT +#define CK_F_PR_BTC_INT +CK_PR_BTX_S(btc, int, int, ^,) +#endif /* CK_F_PR_BTC_INT */ + +#ifndef CK_F_PR_BTR_INT +#define CK_F_PR_BTR_INT +CK_PR_BTX_S(btr, int, int, &, ~) +#endif /* CK_F_PR_BTR_INT */ + +#ifndef CK_F_PR_BTS_INT +#define CK_F_PR_BTS_INT +CK_PR_BTX_S(bts, int, int, |,) +#endif /* CK_F_PR_BTS_INT */ + +#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ + +#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) + +#ifndef CK_F_PR_BTC_UINT +#define CK_F_PR_BTC_UINT +CK_PR_BTX_S(btc, uint, unsigned int, ^,) +#endif /* CK_F_PR_BTC_UINT */ + +#ifndef CK_F_PR_BTR_UINT +#define CK_F_PR_BTR_UINT +CK_PR_BTX_S(btr, uint, unsigned int, &, ~) +#endif /* CK_F_PR_BTR_UINT */ + +#ifndef CK_F_PR_BTS_UINT +#define CK_F_PR_BTS_UINT +CK_PR_BTX_S(bts, uint, unsigned int, |,) +#endif /* CK_F_PR_BTS_UINT */ + +#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ + +#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) + +#ifndef CK_F_PR_BTC_PTR +#define CK_F_PR_BTC_PTR +CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,) +#endif /* CK_F_PR_BTC_PTR */ + +#ifndef CK_F_PR_BTR_PTR +#define CK_F_PR_BTR_PTR +CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~) +#endif /* CK_F_PR_BTR_PTR */ + +#ifndef CK_F_PR_BTS_PTR +#define CK_F_PR_BTS_PTR +CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,) +#endif /* CK_F_PR_BTS_PTR */ + +#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ + +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) + +#ifndef CK_F_PR_BTC_64 +#define CK_F_PR_BTC_64 +CK_PR_BTX_S(btc, 64, uint64_t, ^,) +#endif /* CK_F_PR_BTC_64 */ + +#ifndef CK_F_PR_BTR_64 +#define CK_F_PR_BTR_64 +CK_PR_BTX_S(btr, 64, uint64_t, &, ~) +#endif /* CK_F_PR_BTR_64 */ + +#ifndef CK_F_PR_BTS_64 +#define CK_F_PR_BTS_64 +CK_PR_BTX_S(bts, 64, uint64_t, |,) +#endif /* CK_F_PR_BTS_64 */ + +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ + +#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) + +#ifndef CK_F_PR_BTC_32 +#define CK_F_PR_BTC_32 +CK_PR_BTX_S(btc, 32, uint32_t, ^,) +#endif /* CK_F_PR_BTC_32 */ + +#ifndef CK_F_PR_BTR_32 +#define CK_F_PR_BTR_32 +CK_PR_BTX_S(btr, 32, uint32_t, &, ~) +#endif /* CK_F_PR_BTR_32 */ + +#ifndef CK_F_PR_BTS_32 +#define CK_F_PR_BTS_32 +CK_PR_BTX_S(bts, 32, uint32_t, |,) +#endif /* CK_F_PR_BTS_32 */ + +#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ + +#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) + +#ifndef CK_F_PR_BTC_16 +#define CK_F_PR_BTC_16 +CK_PR_BTX_S(btc, 16, uint16_t, ^,) +#endif /* CK_F_PR_BTC_16 */ + +#ifndef CK_F_PR_BTR_16 +#define CK_F_PR_BTR_16 +CK_PR_BTX_S(btr, 16, uint16_t, &, ~) +#endif /* CK_F_PR_BTR_16 */ + +#ifndef CK_F_PR_BTS_16 +#define CK_F_PR_BTS_16 +CK_PR_BTX_S(bts, 16, uint16_t, |,) +#endif /* CK_F_PR_BTS_16 */ + +#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ + +#undef CK_PR_BTX_S +#undef CK_PR_BTX + +#define CK_PR_UNARY(K, X, S, M, T) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target) \ + { \ + ck_pr_##X##_##S(target, (T)1); \ + return; \ + } + +#define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S##_zero(M *target, bool *zero) \ + { \ + T previous; \ + C punt; \ + punt = (C)ck_pr_md_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(previous P 1), \ + &previous) == false) \ + ck_pr_stall(); \ + *zero = previous == (T)Z; \ + return; \ + } + +#define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M) +#define CK_PR_UNARY_Z_S(K, S, M, P, Z) CK_PR_UNARY_Z(K, S, M, M, P, M, Z) + +#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) + +#ifndef CK_F_PR_INC_CHAR +#define CK_F_PR_INC_CHAR +CK_PR_UNARY_S(inc, add, char, char) +#endif /* CK_F_PR_INC_CHAR */ + +#ifndef CK_F_PR_INC_CHAR_ZERO +#define CK_F_PR_INC_CHAR_ZERO +CK_PR_UNARY_Z_S(inc, char, char, +, -1) +#endif /* CK_F_PR_INC_CHAR_ZERO */ + +#ifndef CK_F_PR_DEC_CHAR +#define CK_F_PR_DEC_CHAR +CK_PR_UNARY_S(dec, sub, char, char) +#endif /* CK_F_PR_DEC_CHAR */ + +#ifndef CK_F_PR_DEC_CHAR_ZERO +#define CK_F_PR_DEC_CHAR_ZERO +CK_PR_UNARY_Z_S(dec, char, char, -, 1) +#endif /* CK_F_PR_DEC_CHAR_ZERO */ + +#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ + +#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) + +#ifndef CK_F_PR_INC_INT +#define CK_F_PR_INC_INT +CK_PR_UNARY_S(inc, add, int, int) +#endif /* CK_F_PR_INC_INT */ + +#ifndef CK_F_PR_INC_INT_ZERO +#define CK_F_PR_INC_INT_ZERO +CK_PR_UNARY_Z_S(inc, int, int, +, -1) +#endif /* CK_F_PR_INC_INT_ZERO */ + +#ifndef CK_F_PR_DEC_INT +#define CK_F_PR_DEC_INT +CK_PR_UNARY_S(dec, sub, int, int) +#endif /* CK_F_PR_DEC_INT */ + +#ifndef CK_F_PR_DEC_INT_ZERO +#define CK_F_PR_DEC_INT_ZERO +CK_PR_UNARY_Z_S(dec, int, int, -, 1) +#endif /* CK_F_PR_DEC_INT_ZERO */ + +#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ + +#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ + !defined(CK_PR_DISABLE_DOUBLE) + +#ifndef CK_F_PR_INC_DOUBLE +#define CK_F_PR_INC_DOUBLE +CK_PR_UNARY_S(inc, add, double, double) +#endif /* CK_F_PR_INC_DOUBLE */ + +#ifndef CK_F_PR_DEC_DOUBLE +#define CK_F_PR_DEC_DOUBLE +CK_PR_UNARY_S(dec, sub, double, double) +#endif /* CK_F_PR_DEC_DOUBLE */ + +#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ + +#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) + +#ifndef CK_F_PR_INC_UINT +#define CK_F_PR_INC_UINT +CK_PR_UNARY_S(inc, add, uint, unsigned int) +#endif /* CK_F_PR_INC_UINT */ + +#ifndef CK_F_PR_INC_UINT_ZERO +#define CK_F_PR_INC_UINT_ZERO +CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX) +#endif /* CK_F_PR_INC_UINT_ZERO */ + +#ifndef CK_F_PR_DEC_UINT +#define CK_F_PR_DEC_UINT +CK_PR_UNARY_S(dec, sub, uint, unsigned int) +#endif /* CK_F_PR_DEC_UINT */ + +#ifndef CK_F_PR_DEC_UINT_ZERO +#define CK_F_PR_DEC_UINT_ZERO +CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1) +#endif /* CK_F_PR_DEC_UINT_ZERO */ + +#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ + +#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) + +#ifndef CK_F_PR_INC_PTR +#define CK_F_PR_INC_PTR +CK_PR_UNARY(inc, add, ptr, void, uintptr_t) +#endif /* CK_F_PR_INC_PTR */ + +#ifndef CK_F_PR_INC_PTR_ZERO +#define CK_F_PR_INC_PTR_ZERO +CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX) +#endif /* CK_F_PR_INC_PTR_ZERO */ + +#ifndef CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_PTR +CK_PR_UNARY(dec, sub, ptr, void, uintptr_t) +#endif /* CK_F_PR_DEC_PTR */ + +#ifndef CK_F_PR_DEC_PTR_ZERO +#define CK_F_PR_DEC_PTR_ZERO +CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1) +#endif /* CK_F_PR_DEC_PTR_ZERO */ + +#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ + +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) + +#ifndef CK_F_PR_INC_64 +#define CK_F_PR_INC_64 +CK_PR_UNARY_S(inc, add, 64, uint64_t) +#endif /* CK_F_PR_INC_64 */ + +#ifndef CK_F_PR_INC_64_ZERO +#define CK_F_PR_INC_64_ZERO +CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX) +#endif /* CK_F_PR_INC_64_ZERO */ + +#ifndef CK_F_PR_DEC_64 +#define CK_F_PR_DEC_64 +CK_PR_UNARY_S(dec, sub, 64, uint64_t) +#endif /* CK_F_PR_DEC_64 */ + +#ifndef CK_F_PR_DEC_64_ZERO +#define CK_F_PR_DEC_64_ZERO +CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1) +#endif /* CK_F_PR_DEC_64_ZERO */ + +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ + +#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) + +#ifndef CK_F_PR_INC_32 +#define CK_F_PR_INC_32 +CK_PR_UNARY_S(inc, add, 32, uint32_t) +#endif /* CK_F_PR_INC_32 */ + +#ifndef CK_F_PR_INC_32_ZERO +#define CK_F_PR_INC_32_ZERO +CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX) +#endif /* CK_F_PR_INC_32_ZERO */ + +#ifndef CK_F_PR_DEC_32 +#define CK_F_PR_DEC_32 +CK_PR_UNARY_S(dec, sub, 32, uint32_t) +#endif /* CK_F_PR_DEC_32 */ + +#ifndef CK_F_PR_DEC_32_ZERO +#define CK_F_PR_DEC_32_ZERO +CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1) +#endif /* CK_F_PR_DEC_32_ZERO */ + +#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ + +#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) + +#ifndef CK_F_PR_INC_16 +#define CK_F_PR_INC_16 +CK_PR_UNARY_S(inc, add, 16, uint16_t) +#endif /* CK_F_PR_INC_16 */ + +#ifndef CK_F_PR_INC_16_ZERO +#define CK_F_PR_INC_16_ZERO +CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX) +#endif /* CK_F_PR_INC_16_ZERO */ + +#ifndef CK_F_PR_DEC_16 +#define CK_F_PR_DEC_16 +CK_PR_UNARY_S(dec, sub, 16, uint16_t) +#endif /* CK_F_PR_DEC_16 */ + +#ifndef CK_F_PR_DEC_16_ZERO +#define CK_F_PR_DEC_16_ZERO +CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1) +#endif /* CK_F_PR_DEC_16_ZERO */ + +#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ + +#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) + +#ifndef CK_F_PR_INC_8 +#define CK_F_PR_INC_8 +CK_PR_UNARY_S(inc, add, 8, uint8_t) +#endif /* CK_F_PR_INC_8 */ + +#ifndef CK_F_PR_INC_8_ZERO +#define CK_F_PR_INC_8_ZERO +CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX) +#endif /* CK_F_PR_INC_8_ZERO */ + +#ifndef CK_F_PR_DEC_8 +#define CK_F_PR_DEC_8 +CK_PR_UNARY_S(dec, sub, 8, uint8_t) +#endif /* CK_F_PR_DEC_8 */ + +#ifndef CK_F_PR_DEC_8_ZERO +#define CK_F_PR_DEC_8_ZERO +CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1) +#endif /* CK_F_PR_DEC_8_ZERO */ + +#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ + +#undef CK_PR_UNARY_Z_S +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY_Z +#undef CK_PR_UNARY + +#define CK_PR_N(K, S, M, T, P, C) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target) \ + { \ + T previous; \ + C punt; \ + punt = (C)ck_pr_md_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(P previous), \ + &previous) == false) \ + ck_pr_stall(); \ + \ + return; \ + } + +#define CK_PR_N_Z(S, M, T, C) \ + CK_CC_INLINE static void \ + ck_pr_neg_##S##_zero(M *target, bool *zero) \ + { \ + T previous; \ + C punt; \ + punt = (C)ck_pr_md_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(-previous), \ + &previous) == false) \ + ck_pr_stall(); \ + \ + *zero = previous == 0; \ + return; \ + } + +#define CK_PR_N_S(K, S, M, P) CK_PR_N(K, S, M, M, P, M) +#define CK_PR_N_Z_S(S, M) CK_PR_N_Z(S, M, M, M) + +#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) + +#ifndef CK_F_PR_NOT_CHAR +#define CK_F_PR_NOT_CHAR +CK_PR_N_S(not, char, char, ~) +#endif /* CK_F_PR_NOT_CHAR */ + +#ifndef CK_F_PR_NEG_CHAR +#define CK_F_PR_NEG_CHAR +CK_PR_N_S(neg, char, char, -) +#endif /* CK_F_PR_NEG_CHAR */ + +#ifndef CK_F_PR_NEG_CHAR_ZERO +#define CK_F_PR_NEG_CHAR_ZERO +CK_PR_N_Z_S(char, char) +#endif /* CK_F_PR_NEG_CHAR_ZERO */ + +#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ + +#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) + +#ifndef CK_F_PR_NOT_INT +#define CK_F_PR_NOT_INT +CK_PR_N_S(not, int, int, ~) +#endif /* CK_F_PR_NOT_INT */ + +#ifndef CK_F_PR_NEG_INT +#define CK_F_PR_NEG_INT +CK_PR_N_S(neg, int, int, -) +#endif /* CK_F_PR_NEG_INT */ + +#ifndef CK_F_PR_NEG_INT_ZERO +#define CK_F_PR_NEG_INT_ZERO +CK_PR_N_Z_S(int, int) +#endif /* CK_F_PR_NEG_INT_ZERO */ + +#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ + +#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ + !defined(CK_PR_DISABLE_DOUBLE) + +#ifndef CK_F_PR_NEG_DOUBLE +#define CK_F_PR_NEG_DOUBLE +CK_PR_N_S(neg, double, double, -) +#endif /* CK_F_PR_NEG_DOUBLE */ + +#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ + +#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) + +#ifndef CK_F_PR_NOT_UINT +#define CK_F_PR_NOT_UINT +CK_PR_N_S(not, uint, unsigned int, ~) +#endif /* CK_F_PR_NOT_UINT */ + +#ifndef CK_F_PR_NEG_UINT +#define CK_F_PR_NEG_UINT +CK_PR_N_S(neg, uint, unsigned int, -) +#endif /* CK_F_PR_NEG_UINT */ + +#ifndef CK_F_PR_NEG_UINT_ZERO +#define CK_F_PR_NEG_UINT_ZERO +CK_PR_N_Z_S(uint, unsigned int) +#endif /* CK_F_PR_NEG_UINT_ZERO */ + +#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ + +#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) + +#ifndef CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_PTR +CK_PR_N(not, ptr, void, uintptr_t, ~, void *) +#endif /* CK_F_PR_NOT_PTR */ + +#ifndef CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_PTR +CK_PR_N(neg, ptr, void, uintptr_t, -, void *) +#endif /* CK_F_PR_NEG_PTR */ + +#ifndef CK_F_PR_NEG_PTR_ZERO +#define CK_F_PR_NEG_PTR_ZERO +CK_PR_N_Z(ptr, void, uintptr_t, void *) +#endif /* CK_F_PR_NEG_PTR_ZERO */ + +#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ + +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) + +#ifndef CK_F_PR_NOT_64 +#define CK_F_PR_NOT_64 +CK_PR_N_S(not, 64, uint64_t, ~) +#endif /* CK_F_PR_NOT_64 */ + +#ifndef CK_F_PR_NEG_64 +#define CK_F_PR_NEG_64 +CK_PR_N_S(neg, 64, uint64_t, -) +#endif /* CK_F_PR_NEG_64 */ + +#ifndef CK_F_PR_NEG_64_ZERO +#define CK_F_PR_NEG_64_ZERO +CK_PR_N_Z_S(64, uint64_t) +#endif /* CK_F_PR_NEG_64_ZERO */ + +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ + +#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) + +#ifndef CK_F_PR_NOT_32 +#define CK_F_PR_NOT_32 +CK_PR_N_S(not, 32, uint32_t, ~) +#endif /* CK_F_PR_NOT_32 */ + +#ifndef CK_F_PR_NEG_32 +#define CK_F_PR_NEG_32 +CK_PR_N_S(neg, 32, uint32_t, -) +#endif /* CK_F_PR_NEG_32 */ + +#ifndef CK_F_PR_NEG_32_ZERO +#define CK_F_PR_NEG_32_ZERO +CK_PR_N_Z_S(32, uint32_t) +#endif /* CK_F_PR_NEG_32_ZERO */ + +#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ + +#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) + +#ifndef CK_F_PR_NOT_16 +#define CK_F_PR_NOT_16 +CK_PR_N_S(not, 16, uint16_t, ~) +#endif /* CK_F_PR_NOT_16 */ + +#ifndef CK_F_PR_NEG_16 +#define CK_F_PR_NEG_16 +CK_PR_N_S(neg, 16, uint16_t, -) +#endif /* CK_F_PR_NEG_16 */ + +#ifndef CK_F_PR_NEG_16_ZERO +#define CK_F_PR_NEG_16_ZERO +CK_PR_N_Z_S(16, uint16_t) +#endif /* CK_F_PR_NEG_16_ZERO */ + +#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ + +#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) + +#ifndef CK_F_PR_NOT_8 +#define CK_F_PR_NOT_8 +CK_PR_N_S(not, 8, uint8_t, ~) +#endif /* CK_F_PR_NOT_8 */ + +#ifndef CK_F_PR_NEG_8 +#define CK_F_PR_NEG_8 +CK_PR_N_S(neg, 8, uint8_t, -) +#endif /* CK_F_PR_NEG_8 */ + +#ifndef CK_F_PR_NEG_8_ZERO +#define CK_F_PR_NEG_8_ZERO +CK_PR_N_Z_S(8, uint8_t) +#endif /* CK_F_PR_NEG_8_ZERO */ + +#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ + +#undef CK_PR_N_Z_S +#undef CK_PR_N_S +#undef CK_PR_N_Z +#undef CK_PR_N + +#define CK_PR_FAA(S, M, T, C) \ + CK_CC_INLINE static C \ + ck_pr_faa_##S(M *target, T delta) \ + { \ + T previous; \ + C punt; \ + punt = (C)ck_pr_md_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(previous + delta), \ + &previous) == false) \ + ck_pr_stall(); \ + \ + return ((C)previous); \ + } + +#define CK_PR_FAS(S, M, C) \ + CK_CC_INLINE static C \ + ck_pr_fas_##S(M *target, C update) \ + { \ + C previous; \ + previous = ck_pr_md_load_##S(target); \ + while (ck_pr_cas_##S##_value(target, \ + previous, \ + update, \ + &previous) == false) \ + ck_pr_stall(); \ + \ + return (previous); \ + } + +#define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M) +#define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M) + +#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) + +#ifndef CK_F_PR_FAA_CHAR +#define CK_F_PR_FAA_CHAR +CK_PR_FAA_S(char, char) +#endif /* CK_F_PR_FAA_CHAR */ + +#ifndef CK_F_PR_FAS_CHAR +#define CK_F_PR_FAS_CHAR +CK_PR_FAS_S(char, char) +#endif /* CK_F_PR_FAS_CHAR */ + +#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ + +#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) + +#ifndef CK_F_PR_FAA_INT +#define CK_F_PR_FAA_INT +CK_PR_FAA_S(int, int) +#endif /* CK_F_PR_FAA_INT */ + +#ifndef CK_F_PR_FAS_INT +#define CK_F_PR_FAS_INT +CK_PR_FAS_S(int, int) +#endif /* CK_F_PR_FAS_INT */ + +#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ + +#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ + !defined(CK_PR_DISABLE_DOUBLE) + +#ifndef CK_F_PR_FAA_DOUBLE +#define CK_F_PR_FAA_DOUBLE +CK_PR_FAA_S(double, double) +#endif /* CK_F_PR_FAA_DOUBLE */ + +#ifndef CK_F_PR_FAS_DOUBLE +#define CK_F_PR_FAS_DOUBLE +CK_PR_FAS_S(double, double) +#endif /* CK_F_PR_FAS_DOUBLE */ + +#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ + +#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) + +#ifndef CK_F_PR_FAA_UINT +#define CK_F_PR_FAA_UINT +CK_PR_FAA_S(uint, unsigned int) +#endif /* CK_F_PR_FAA_UINT */ + +#ifndef CK_F_PR_FAS_UINT +#define CK_F_PR_FAS_UINT +CK_PR_FAS_S(uint, unsigned int) +#endif /* CK_F_PR_FAS_UINT */ + +#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ + +#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) + +#ifndef CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_PTR +CK_PR_FAA(ptr, void, uintptr_t, void *) +#endif /* CK_F_PR_FAA_PTR */ + +#ifndef CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_PTR +CK_PR_FAS(ptr, void, void *) +#endif /* CK_F_PR_FAS_PTR */ + +#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ + +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) + +#ifndef CK_F_PR_FAA_64 +#define CK_F_PR_FAA_64 +CK_PR_FAA_S(64, uint64_t) +#endif /* CK_F_PR_FAA_64 */ + +#ifndef CK_F_PR_FAS_64 +#define CK_F_PR_FAS_64 +CK_PR_FAS_S(64, uint64_t) +#endif /* CK_F_PR_FAS_64 */ + +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ + +#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) + +#ifndef CK_F_PR_FAA_32 +#define CK_F_PR_FAA_32 +CK_PR_FAA_S(32, uint32_t) +#endif /* CK_F_PR_FAA_32 */ + +#ifndef CK_F_PR_FAS_32 +#define CK_F_PR_FAS_32 +CK_PR_FAS_S(32, uint32_t) +#endif /* CK_F_PR_FAS_32 */ + +#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ + +#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) + +#ifndef CK_F_PR_FAA_16 +#define CK_F_PR_FAA_16 +CK_PR_FAA_S(16, uint16_t) +#endif /* CK_F_PR_FAA_16 */ + +#ifndef CK_F_PR_FAS_16 +#define CK_F_PR_FAS_16 +CK_PR_FAS_S(16, uint16_t) +#endif /* CK_F_PR_FAS_16 */ + +#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ + +#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) + +#ifndef CK_F_PR_FAA_8 +#define CK_F_PR_FAA_8 +CK_PR_FAA_S(8, uint8_t) +#endif /* CK_F_PR_FAA_8 */ + +#ifndef CK_F_PR_FAS_8 +#define CK_F_PR_FAS_8 +CK_PR_FAS_S(8, uint8_t) +#endif /* CK_F_PR_FAS_8 */ + +#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ + +#undef CK_PR_FAA_S +#undef CK_PR_FAS_S +#undef CK_PR_FAA +#undef CK_PR_FAS + +#endif /* CK_PR_H */ diff --git a/freebsd/sys/contrib/ck/include/ck_queue.h b/freebsd/sys/contrib/ck/include/ck_queue.h new file mode 100644 index 00000000..faf96a17 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/ck_queue.h @@ -0,0 +1,428 @@ +/* + * Copyright 2012-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/*- + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)queue.h 8.5 (Berkeley) 8/20/94 + * $FreeBSD$ + */ + +#ifndef CK_QUEUE_H +#define CK_QUEUE_H + +#include <ck_pr.h> + +/* + * This file defines three types of data structures: singly-linked lists, + * singly-linked tail queues and lists. + * + * A singly-linked list is headed by a single forward pointer. The elements + * are singly linked for minimum space and pointer manipulation overhead at + * the expense of O(n) removal for arbitrary elements. New elements can be + * added to the list after an existing element or at the head of the list. + * Elements being removed from the head of the list should use the explicit + * macro for this purpose for optimum efficiency. A singly-linked list may + * only be traversed in the forward direction. Singly-linked lists are ideal + * for applications with large datasets and few or no removals or for + * implementing a LIFO queue. + * + * A singly-linked tail queue is headed by a pair of pointers, one to the + * head of the list and the other to the tail of the list. The elements are + * singly linked for minimum space and pointer manipulation overhead at the + * expense of O(n) removal for arbitrary elements. New elements can be added + * to the list after an existing element, at the head of the list, or at the + * end of the list. Elements being removed from the head of the tail queue + * should use the explicit macro for this purpose for optimum efficiency. + * A singly-linked tail queue may only be traversed in the forward direction. + * Singly-linked tail queues are ideal for applications with large datasets + * and few or no removals or for implementing a FIFO queue. + * + * A list is headed by a single forward pointer (or an array of forward + * pointers for a hash table header). The elements are doubly linked + * so that an arbitrary element can be removed without a need to + * traverse the list. New elements can be added to the list before + * or after an existing element or at the head of the list. A list + * may only be traversed in the forward direction. + * + * It is safe to use _FOREACH/_FOREACH_SAFE in the presence of concurrent + * modifications to the list. Writers to these lists must, on the other hand, + * implement writer-side synchronization. The _SWAP operations are not atomic. + * This facility is currently unsupported on architectures such as the Alpha + * which require load-depend memory fences. + * + * CK_SLIST CK_LIST CK_STAILQ + * _HEAD + + + + * _HEAD_INITIALIZER + + + + * _ENTRY + + + + * _INIT + + + + * _EMPTY + + + + * _FIRST + + + + * _NEXT + + + + * _FOREACH + + + + * _FOREACH_SAFE + + + + * _INSERT_HEAD + + + + * _INSERT_BEFORE - + - + * _INSERT_AFTER + + + + * _INSERT_TAIL - - + + * _REMOVE_AFTER + - + + * _REMOVE_HEAD + - + + * _REMOVE + + + + * _SWAP + + + + * _MOVE + + + + */ + +/* + * Singly-linked List declarations. + */ +#define CK_SLIST_HEAD(name, type) \ +struct name { \ + struct type *cslh_first; /* first element */ \ +} + +#define CK_SLIST_HEAD_INITIALIZER(head) \ + { NULL } + +#define CK_SLIST_ENTRY(type) \ +struct { \ + struct type *csle_next; /* next element */ \ +} + +/* + * Singly-linked List functions. + */ +#define CK_SLIST_EMPTY(head) \ + (ck_pr_load_ptr(&(head)->cslh_first) == NULL) + +#define CK_SLIST_FIRST(head) \ + (ck_pr_load_ptr(&(head)->cslh_first)) + +#define CK_SLIST_NEXT(elm, field) \ + ck_pr_load_ptr(&((elm)->field.csle_next)) + +#define CK_SLIST_FOREACH(var, head, field) \ + for ((var) = CK_SLIST_FIRST((head)); \ + (var) && (ck_pr_fence_load(), 1); \ + (var) = CK_SLIST_NEXT((var), field)) + +#define CK_SLIST_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = CK_SLIST_FIRST(head); \ + (var) && (ck_pr_fence_load(), (tvar) = CK_SLIST_NEXT(var, field), 1);\ + (var) = (tvar)) + +#define CK_SLIST_FOREACH_PREVPTR(var, varp, head, field) \ + for ((varp) = &(head)->cslh_first; \ + ((var) = ck_pr_load_ptr(varp)) != NULL && (ck_pr_fence_load(), 1); \ + (varp) = &(var)->field.csle_next) + +#define CK_SLIST_INIT(head) do { \ + ck_pr_store_ptr(&(head)->cslh_first, NULL); \ + ck_pr_fence_store(); \ +} while (0) + +#define CK_SLIST_INSERT_AFTER(a, b, field) do { \ + (b)->field.csle_next = (a)->field.csle_next; \ + ck_pr_fence_store(); \ + ck_pr_store_ptr(&(a)->field.csle_next, b); \ +} while (0) + +#define CK_SLIST_INSERT_HEAD(head, elm, field) do { \ + (elm)->field.csle_next = (head)->cslh_first; \ + ck_pr_fence_store(); \ + ck_pr_store_ptr(&(head)->cslh_first, elm); \ +} while (0) + +#define CK_SLIST_REMOVE_AFTER(elm, field) do { \ + ck_pr_store_ptr(&(elm)->field.csle_next, \ + (elm)->field.csle_next->field.csle_next); \ +} while (0) + +#define CK_SLIST_REMOVE(head, elm, type, field) do { \ + if ((head)->cslh_first == (elm)) { \ + CK_SLIST_REMOVE_HEAD((head), field); \ + } else { \ + struct type *curelm = (head)->cslh_first; \ + while (curelm->field.csle_next != (elm)) \ + curelm = curelm->field.csle_next; \ + CK_SLIST_REMOVE_AFTER(curelm, field); \ + } \ +} while (0) + +#define CK_SLIST_REMOVE_HEAD(head, field) do { \ + ck_pr_store_ptr(&(head)->cslh_first, \ + (head)->cslh_first->field.csle_next); \ +} while (0) + +#define CK_SLIST_MOVE(head1, head2, field) do { \ + ck_pr_store_ptr(&(head1)->cslh_first, (head2)->cslh_first); \ +} while (0) + +/* + * This operation is not applied atomically. + */ +#define CK_SLIST_SWAP(a, b, type) do { \ + struct type *swap_first = (a)->cslh_first; \ + (a)->cslh_first = (b)->cslh_first; \ + (b)->cslh_first = swap_first; \ +} while (0) + +/* + * Singly-linked Tail queue declarations. + */ +#define CK_STAILQ_HEAD(name, type) \ +struct name { \ + struct type *cstqh_first;/* first element */ \ + struct type **cstqh_last;/* addr of last next element */ \ +} + +#define CK_STAILQ_HEAD_INITIALIZER(head) \ + { NULL, &(head).cstqh_first } + +#define CK_STAILQ_ENTRY(type) \ +struct { \ + struct type *cstqe_next; /* next element */ \ +} + +/* + * Singly-linked Tail queue functions. + */ +#define CK_STAILQ_CONCAT(head1, head2) do { \ + if ((head2)->cstqh_first != NULL) { \ + ck_pr_store_ptr((head1)->cstqh_last, (head2)->cstqh_first); \ + ck_pr_fence_store(); \ + (head1)->cstqh_last = (head2)->cstqh_last; \ + CK_STAILQ_INIT((head2)); \ + } \ +} while (0) + +#define CK_STAILQ_EMPTY(head) (ck_pr_load_ptr(&(head)->cstqh_first) == NULL) + +#define CK_STAILQ_FIRST(head) (ck_pr_load_ptr(&(head)->cstqh_first)) + +#define CK_STAILQ_FOREACH(var, head, field) \ + for((var) = CK_STAILQ_FIRST((head)); \ + (var) && (ck_pr_fence_load(), 1); \ + (var) = CK_STAILQ_NEXT((var), field)) + +#define CK_STAILQ_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = CK_STAILQ_FIRST((head)); \ + (var) && (ck_pr_fence_load(), (tvar) = \ + CK_STAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define CK_STAILQ_INIT(head) do { \ + ck_pr_store_ptr(&(head)->cstqh_first, NULL); \ + ck_pr_fence_store(); \ + (head)->cstqh_last = &(head)->cstqh_first; \ +} while (0) + +#define CK_STAILQ_INSERT_AFTER(head, tqelm, elm, field) do { \ + (elm)->field.cstqe_next = (tqelm)->field.cstqe_next; \ + ck_pr_fence_store(); \ + ck_pr_store_ptr(&(tqelm)->field.cstqe_next, elm); \ + if ((elm)->field.cstqe_next == NULL) \ + (head)->cstqh_last = &(elm)->field.cstqe_next; \ +} while (0) + +#define CK_STAILQ_INSERT_HEAD(head, elm, field) do { \ + (elm)->field.cstqe_next = (head)->cstqh_first; \ + ck_pr_fence_store(); \ + ck_pr_store_ptr(&(head)->cstqh_first, elm); \ + if ((elm)->field.cstqe_next == NULL) \ + (head)->cstqh_last = &(elm)->field.cstqe_next; \ +} while (0) + +#define CK_STAILQ_INSERT_TAIL(head, elm, field) do { \ + (elm)->field.cstqe_next = NULL; \ + ck_pr_fence_store(); \ + ck_pr_store_ptr((head)->cstqh_last, (elm)); \ + (head)->cstqh_last = &(elm)->field.cstqe_next; \ +} while (0) + +#define CK_STAILQ_NEXT(elm, field) \ + (ck_pr_load_ptr(&(elm)->field.cstqe_next)) + +#define CK_STAILQ_REMOVE(head, elm, type, field) do { \ + if ((head)->cstqh_first == (elm)) { \ + CK_STAILQ_REMOVE_HEAD((head), field); \ + } else { \ + struct type *curelm = (head)->cstqh_first; \ + while (curelm->field.cstqe_next != (elm)) \ + curelm = curelm->field.cstqe_next; \ + CK_STAILQ_REMOVE_AFTER(head, curelm, field); \ + } \ +} while (0) + +#define CK_STAILQ_REMOVE_AFTER(head, elm, field) do { \ + ck_pr_store_ptr(&(elm)->field.cstqe_next, \ + (elm)->field.cstqe_next->field.cstqe_next); \ + if ((elm)->field.cstqe_next == NULL) \ + (head)->cstqh_last = &(elm)->field.cstqe_next; \ +} while (0) + +#define CK_STAILQ_REMOVE_HEAD(head, field) do { \ + ck_pr_store_ptr(&(head)->cstqh_first, \ + (head)->cstqh_first->field.cstqe_next); \ + if ((head)->cstqh_first == NULL) \ + (head)->cstqh_last = &(head)->cstqh_first; \ +} while (0) + +#define CK_STAILQ_MOVE(head1, head2, field) do { \ + ck_pr_store_ptr(&(head1)->cstqh_first, (head2)->cstqh_first); \ + (head1)->cstqh_last = (head2)->cstqh_last; \ + if ((head2)->cstqh_last == &(head2)->cstqh_first) \ + (head1)->cstqh_last = &(head1)->cstqh_first; \ +} while (0) + +/* + * This operation is not applied atomically. + */ +#define CK_STAILQ_SWAP(head1, head2, type) do { \ + struct type *swap_first = CK_STAILQ_FIRST(head1); \ + struct type **swap_last = (head1)->cstqh_last; \ + CK_STAILQ_FIRST(head1) = CK_STAILQ_FIRST(head2); \ + (head1)->cstqh_last = (head2)->cstqh_last; \ + CK_STAILQ_FIRST(head2) = swap_first; \ + (head2)->cstqh_last = swap_last; \ + if (CK_STAILQ_EMPTY(head1)) \ + (head1)->cstqh_last = &(head1)->cstqh_first; \ + if (CK_STAILQ_EMPTY(head2)) \ + (head2)->cstqh_last = &(head2)->cstqh_first; \ +} while (0) + +/* + * List declarations. + */ +#define CK_LIST_HEAD(name, type) \ +struct name { \ + struct type *clh_first; /* first element */ \ +} + +#define CK_LIST_HEAD_INITIALIZER(head) \ + { NULL } + +#define CK_LIST_ENTRY(type) \ +struct { \ + struct type *cle_next; /* next element */ \ + struct type **cle_prev; /* address of previous next element */ \ +} + +#define CK_LIST_FIRST(head) ck_pr_load_ptr(&(head)->clh_first) +#define CK_LIST_EMPTY(head) (CK_LIST_FIRST(head) == NULL) +#define CK_LIST_NEXT(elm, field) ck_pr_load_ptr(&(elm)->field.cle_next) + +#define CK_LIST_FOREACH(var, head, field) \ + for ((var) = CK_LIST_FIRST((head)); \ + (var) && (ck_pr_fence_load(), 1); \ + (var) = CK_LIST_NEXT((var), field)) + +#define CK_LIST_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = CK_LIST_FIRST((head)); \ + (var) && (ck_pr_fence_load(), (tvar) = CK_LIST_NEXT((var), field), 1);\ + (var) = (tvar)) + +#define CK_LIST_INIT(head) do { \ + ck_pr_store_ptr(&(head)->clh_first, NULL); \ + ck_pr_fence_store(); \ +} while (0) + +#define CK_LIST_INSERT_AFTER(listelm, elm, field) do { \ + (elm)->field.cle_next = (listelm)->field.cle_next; \ + (elm)->field.cle_prev = &(listelm)->field.cle_next; \ + ck_pr_fence_store(); \ + if ((listelm)->field.cle_next != NULL) \ + (listelm)->field.cle_next->field.cle_prev = &(elm)->field.cle_next;\ + ck_pr_store_ptr(&(listelm)->field.cle_next, elm); \ +} while (0) + +#define CK_LIST_INSERT_BEFORE(listelm, elm, field) do { \ + (elm)->field.cle_prev = (listelm)->field.cle_prev; \ + (elm)->field.cle_next = (listelm); \ + ck_pr_fence_store(); \ + ck_pr_store_ptr((listelm)->field.cle_prev, (elm)); \ + (listelm)->field.cle_prev = &(elm)->field.cle_next; \ +} while (0) + +#define CK_LIST_INSERT_HEAD(head, elm, field) do { \ + (elm)->field.cle_next = (head)->clh_first; \ + ck_pr_fence_store(); \ + if ((elm)->field.cle_next != NULL) \ + (head)->clh_first->field.cle_prev = &(elm)->field.cle_next; \ + ck_pr_store_ptr(&(head)->clh_first, elm); \ + (elm)->field.cle_prev = &(head)->clh_first; \ +} while (0) + +#define CK_LIST_REMOVE(elm, field) do { \ + ck_pr_store_ptr((elm)->field.cle_prev, (elm)->field.cle_next); \ + if ((elm)->field.cle_next != NULL) \ + (elm)->field.cle_next->field.cle_prev = (elm)->field.cle_prev; \ +} while (0) + +#define CK_LIST_MOVE(head1, head2, field) do { \ + ck_pr_store_ptr(&(head1)->clh_first, (head2)->clh_first); \ + if ((head1)->clh_first != NULL) \ + (head1)->clh_first->field.cle_prev = &(head1)->clh_first; \ +} while (0) + +/* + * This operation is not applied atomically. + */ +#define CK_LIST_SWAP(head1, head2, type, field) do { \ + struct type *swap_tmp = (head1)->clh_first; \ + (head1)->clh_first = (head2)->clh_first; \ + (head2)->clh_first = swap_tmp; \ + if ((swap_tmp = (head1)->clh_first) != NULL) \ + swap_tmp->field.cle_prev = &(head1)->clh_first; \ + if ((swap_tmp = (head2)->clh_first) != NULL) \ + swap_tmp->field.cle_prev = &(head2)->clh_first; \ +} while (0) + +#endif /* CK_QUEUE_H */ diff --git a/freebsd/sys/contrib/ck/include/ck_stack.h b/freebsd/sys/contrib/ck/include/ck_stack.h new file mode 100644 index 00000000..eb2b685f --- /dev/null +++ b/freebsd/sys/contrib/ck/include/ck_stack.h @@ -0,0 +1,357 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_STACK_H +#define CK_STACK_H + +#include <ck_cc.h> +#include <ck_pr.h> +#include <ck_stdbool.h> +#include <ck_stddef.h> + +struct ck_stack_entry { + struct ck_stack_entry *next; +}; +typedef struct ck_stack_entry ck_stack_entry_t; + +struct ck_stack { + struct ck_stack_entry *head; + char *generation CK_CC_PACKED; +} CK_CC_ALIASED; +typedef struct ck_stack ck_stack_t; + +#define CK_STACK_INITIALIZER { NULL, NULL } + +#ifndef CK_F_STACK_PUSH_UPMC +#define CK_F_STACK_PUSH_UPMC +/* + * Stack producer operation safe for multiple unique producers and multiple consumers. + */ +CK_CC_INLINE static void +ck_stack_push_upmc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + struct ck_stack_entry *stack; + + stack = ck_pr_load_ptr(&target->head); + entry->next = stack; + ck_pr_fence_store(); + + while (ck_pr_cas_ptr_value(&target->head, stack, entry, &stack) == false) { + entry->next = stack; + ck_pr_fence_store(); + } + + return; +} +#endif /* CK_F_STACK_PUSH_UPMC */ + +#ifndef CK_F_STACK_TRYPUSH_UPMC +#define CK_F_STACK_TRYPUSH_UPMC +/* + * Stack producer operation for multiple unique producers and multiple consumers. + * Returns true on success and false on failure. + */ +CK_CC_INLINE static bool +ck_stack_trypush_upmc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + struct ck_stack_entry *stack; + + stack = ck_pr_load_ptr(&target->head); + entry->next = stack; + ck_pr_fence_store(); + + return ck_pr_cas_ptr(&target->head, stack, entry); +} +#endif /* CK_F_STACK_TRYPUSH_UPMC */ + +#ifndef CK_F_STACK_POP_UPMC +#define CK_F_STACK_POP_UPMC +/* + * Stack consumer operation safe for multiple unique producers and multiple consumers. + */ +CK_CC_INLINE static struct ck_stack_entry * +ck_stack_pop_upmc(struct ck_stack *target) +{ + struct ck_stack_entry *entry, *next; + + entry = ck_pr_load_ptr(&target->head); + if (entry == NULL) + return NULL; + + ck_pr_fence_load(); + next = entry->next; + while (ck_pr_cas_ptr_value(&target->head, entry, next, &entry) == false) { + if (entry == NULL) + break; + + ck_pr_fence_load(); + next = entry->next; + } + + return entry; +} +#endif + +#ifndef CK_F_STACK_TRYPOP_UPMC +#define CK_F_STACK_TRYPOP_UPMC +/* + * Stack production operation for multiple unique producers and multiple consumers. + * Returns true on success and false on failure. The value pointed to by the second + * argument is set to a valid ck_stack_entry_t reference if true is returned. If + * false is returned, then the value pointed to by the second argument is undefined. + */ +CK_CC_INLINE static bool +ck_stack_trypop_upmc(struct ck_stack *target, struct ck_stack_entry **r) +{ + struct ck_stack_entry *entry; + + entry = ck_pr_load_ptr(&target->head); + if (entry == NULL) + return false; + + ck_pr_fence_load(); + if (ck_pr_cas_ptr(&target->head, entry, entry->next) == true) { + *r = entry; + return true; + } + + return false; +} +#endif /* CK_F_STACK_TRYPOP_UPMC */ + +#ifndef CK_F_STACK_BATCH_POP_UPMC +#define CK_F_STACK_BATCH_POP_UPMC +/* + * Pop all items off the stack. + */ +CK_CC_INLINE static struct ck_stack_entry * +ck_stack_batch_pop_upmc(struct ck_stack *target) +{ + struct ck_stack_entry *entry; + + entry = ck_pr_fas_ptr(&target->head, NULL); + ck_pr_fence_load(); + return entry; +} +#endif /* CK_F_STACK_BATCH_POP_UPMC */ + +#ifndef CK_F_STACK_PUSH_MPMC +#define CK_F_STACK_PUSH_MPMC +/* + * Stack producer operation safe for multiple producers and multiple consumers. + */ +CK_CC_INLINE static void +ck_stack_push_mpmc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + + ck_stack_push_upmc(target, entry); + return; +} +#endif /* CK_F_STACK_PUSH_MPMC */ + +#ifndef CK_F_STACK_TRYPUSH_MPMC +#define CK_F_STACK_TRYPUSH_MPMC +/* + * Stack producer operation safe for multiple producers and multiple consumers. + */ +CK_CC_INLINE static bool +ck_stack_trypush_mpmc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + + return ck_stack_trypush_upmc(target, entry); +} +#endif /* CK_F_STACK_TRYPUSH_MPMC */ + +#ifdef CK_F_PR_CAS_PTR_2_VALUE +#ifndef CK_F_STACK_POP_MPMC +#define CK_F_STACK_POP_MPMC +/* + * Stack consumer operation safe for multiple producers and multiple consumers. + */ +CK_CC_INLINE static struct ck_stack_entry * +ck_stack_pop_mpmc(struct ck_stack *target) +{ + struct ck_stack original, update; + + original.generation = ck_pr_load_ptr(&target->generation); + ck_pr_fence_load(); + original.head = ck_pr_load_ptr(&target->head); + if (original.head == NULL) + return NULL; + + /* Order with respect to next pointer. */ + ck_pr_fence_load(); + + update.generation = original.generation + 1; + update.head = original.head->next; + + while (ck_pr_cas_ptr_2_value(target, &original, &update, &original) == false) { + if (original.head == NULL) + return NULL; + + update.generation = original.generation + 1; + + /* Order with respect to next pointer. */ + ck_pr_fence_load(); + update.head = original.head->next; + } + + return original.head; +} +#endif /* CK_F_STACK_POP_MPMC */ + +#ifndef CK_F_STACK_TRYPOP_MPMC +#define CK_F_STACK_TRYPOP_MPMC +CK_CC_INLINE static bool +ck_stack_trypop_mpmc(struct ck_stack *target, struct ck_stack_entry **r) +{ + struct ck_stack original, update; + + original.generation = ck_pr_load_ptr(&target->generation); + ck_pr_fence_load(); + original.head = ck_pr_load_ptr(&target->head); + if (original.head == NULL) + return false; + + update.generation = original.generation + 1; + ck_pr_fence_load(); + update.head = original.head->next; + + if (ck_pr_cas_ptr_2_value(target, &original, &update, &original) == true) { + *r = original.head; + return true; + } + + return false; +} +#endif /* CK_F_STACK_TRYPOP_MPMC */ +#endif /* CK_F_PR_CAS_PTR_2_VALUE */ + +#ifndef CK_F_STACK_BATCH_POP_MPMC +#define CK_F_STACK_BATCH_POP_MPMC +/* + * This is equivalent to the UP/MC version as NULL does not need a + * a generation count. + */ +CK_CC_INLINE static struct ck_stack_entry * +ck_stack_batch_pop_mpmc(struct ck_stack *target) +{ + + return ck_stack_batch_pop_upmc(target); +} +#endif /* CK_F_STACK_BATCH_POP_MPMC */ + +#ifndef CK_F_STACK_PUSH_MPNC +#define CK_F_STACK_PUSH_MPNC +/* + * Stack producer operation safe with no concurrent consumers. + */ +CK_CC_INLINE static void +ck_stack_push_mpnc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + struct ck_stack_entry *stack; + + entry->next = NULL; + ck_pr_fence_store_atomic(); + stack = ck_pr_fas_ptr(&target->head, entry); + ck_pr_store_ptr(&entry->next, stack); + ck_pr_fence_store(); + + return; +} +#endif /* CK_F_STACK_PUSH_MPNC */ + +/* + * Stack producer operation for single producer and no concurrent consumers. + */ +CK_CC_INLINE static void +ck_stack_push_spnc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + + entry->next = target->head; + target->head = entry; + return; +} + +/* + * Stack consumer operation for no concurrent producers and single consumer. + */ +CK_CC_INLINE static struct ck_stack_entry * +ck_stack_pop_npsc(struct ck_stack *target) +{ + struct ck_stack_entry *n; + + if (target->head == NULL) + return NULL; + + n = target->head; + target->head = n->next; + + return n; +} + +/* + * Pop all items off a stack. + */ +CK_CC_INLINE static struct ck_stack_entry * +ck_stack_batch_pop_npsc(struct ck_stack *target) +{ + struct ck_stack_entry *n; + + n = target->head; + target->head = NULL; + + return n; +} + +/* + * Stack initialization function. Guarantees initialization across processors. + */ +CK_CC_INLINE static void +ck_stack_init(struct ck_stack *stack) +{ + + stack->head = NULL; + stack->generation = NULL; + return; +} + +/* Defines a container_of functions for */ +#define CK_STACK_CONTAINER(T, M, N) CK_CC_CONTAINER(ck_stack_entry_t, T, M, N) + +#define CK_STACK_ISEMPTY(m) ((m)->head == NULL) +#define CK_STACK_FIRST(s) ((s)->head) +#define CK_STACK_NEXT(m) ((m)->next) +#define CK_STACK_FOREACH(stack, entry) \ + for ((entry) = CK_STACK_FIRST(stack); \ + (entry) != NULL; \ + (entry) = CK_STACK_NEXT(entry)) +#define CK_STACK_FOREACH_SAFE(stack, entry, T) \ + for ((entry) = CK_STACK_FIRST(stack); \ + (entry) != NULL && ((T) = (entry)->next, 1); \ + (entry) = (T)) + +#endif /* CK_STACK_H */ diff --git a/freebsd/sys/contrib/ck/include/ck_stdbool.h b/freebsd/sys/contrib/ck/include/ck_stdbool.h new file mode 100644 index 00000000..b9a79829 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/ck_stdbool.h @@ -0,0 +1,31 @@ +/* + * Copyright 2015 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__FreeBSD__) && defined(_KERNEL) +#include <sys/types.h> +#else +#include <stdbool.h> +#endif diff --git a/freebsd/sys/contrib/ck/include/ck_stddef.h b/freebsd/sys/contrib/ck/include/ck_stddef.h new file mode 100644 index 00000000..6019ea95 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/ck_stddef.h @@ -0,0 +1,31 @@ +/* + * Copyright 2015 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__FreeBSD__) && defined(_KERNEL) +#include <sys/stddef.h> +#else +#include <stddef.h> +#endif diff --git a/freebsd/sys/contrib/ck/include/ck_stdint.h b/freebsd/sys/contrib/ck/include/ck_stdint.h new file mode 100644 index 00000000..8f416a92 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/ck_stdint.h @@ -0,0 +1,34 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__linux__) && defined(__KERNEL__) +#include <linux/kernel.h> +#include <linux/types.h> +#elif defined(__FreeBSD__) && defined(_KERNEL) +#include <sys/stdint.h> +#else +#include <stdint.h> +#endif /* __linux__ && __KERNEL__ */ diff --git a/freebsd/sys/contrib/ck/include/ck_string.h b/freebsd/sys/contrib/ck/include/ck_string.h new file mode 100644 index 00000000..8d2c2525 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/ck_string.h @@ -0,0 +1,31 @@ +/* + * Copyright 2015 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__FreeBSD__) && defined(_KERNEL) +#include <sys/systm.h> +#else +#include <string.h> +#endif diff --git a/freebsd/sys/contrib/ck/include/gcc/aarch64/ck_f_pr.h b/freebsd/sys/contrib/ck/include/gcc/aarch64/ck_f_pr.h new file mode 100644 index 00000000..93ecee07 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/gcc/aarch64/ck_f_pr.h @@ -0,0 +1,167 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_16 +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_64 +#define CK_F_PR_ADD_8 +#define CK_F_PR_ADD_CHAR +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_SHORT +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_16 +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_64 +#define CK_F_PR_AND_8 +#define CK_F_PR_AND_CHAR +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_SHORT +#define CK_F_PR_AND_UINT +#define CK_F_PR_BARRIER +#define CK_F_PR_CAS_16 +#define CK_F_PR_CAS_16_VALUE +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_64 +#define CK_F_PR_CAS_64_VALUE +#define CK_F_PR_CAS_64_2 +#define CK_F_PR_CAS_64_2_VALUE +#define CK_F_PR_CAS_DOUBLE +#define CK_F_PR_CAS_DOUBLE_VALUE +#define CK_F_PR_CAS_8 +#define CK_F_PR_CAS_8_VALUE +#define CK_F_PR_CAS_CHAR +#define CK_F_PR_CAS_CHAR_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_2 +#define CK_F_PR_CAS_PTR_2_VALUE +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_SHORT +#define CK_F_PR_CAS_SHORT_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_16 +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_64 +#define CK_F_PR_DEC_8 +#define CK_F_PR_DEC_CHAR +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_SHORT +#define CK_F_PR_DEC_UINT +#define CK_F_PR_FAA_16 +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_64 +#define CK_F_PR_FAA_8 +#define CK_F_PR_FAA_CHAR +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_SHORT +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FAS_16 +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_64 +#define CK_F_PR_FAS_8 +#define CK_F_PR_FAS_CHAR +#define CK_F_PR_FAS_INT +#define CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_SHORT +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FENCE_ATOMIC +#define CK_F_PR_FENCE_ATOMIC_LOAD +#define CK_F_PR_FENCE_ATOMIC_STORE +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_ATOMIC +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_LOAD_STORE +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STORE_ATOMIC +#define CK_F_PR_FENCE_STORE_LOAD +#define CK_F_PR_FENCE_STRICT_ATOMIC +#define CK_F_PR_FENCE_STRICT_ATOMIC_LOAD +#define CK_F_PR_FENCE_STRICT_ATOMIC_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_LOAD_ATOMIC +#define CK_F_PR_FENCE_STRICT_LOAD_STORE +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_FENCE_STRICT_STORE_ATOMIC +#define CK_F_PR_FENCE_STRICT_STORE_LOAD +#define CK_F_PR_INC_16 +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_64 +#define CK_F_PR_INC_8 +#define CK_F_PR_INC_CHAR +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_SHORT +#define CK_F_PR_INC_UINT +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_64 +#define CK_F_PR_LOAD_DOUBLE +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_SHORT +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_NEG_16 +#define CK_F_PR_NEG_32 +#define CK_F_PR_NEG_64 +#define CK_F_PR_NEG_8 +#define CK_F_PR_NEG_CHAR +#define CK_F_PR_NEG_INT +#define CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_SHORT +#define CK_F_PR_NEG_UINT +#define CK_F_PR_NOT_16 +#define CK_F_PR_NOT_32 +#define CK_F_PR_NOT_64 +#define CK_F_PR_NOT_8 +#define CK_F_PR_NOT_CHAR +#define CK_F_PR_NOT_INT +#define CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_SHORT +#define CK_F_PR_NOT_UINT +#define CK_F_PR_OR_16 +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_64 +#define CK_F_PR_OR_8 +#define CK_F_PR_OR_CHAR +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_SHORT +#define CK_F_PR_OR_UINT +#define CK_F_PR_STALL +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_64 +#define CK_F_PR_STORE_DOUBLE +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_SHORT +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_16 +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_64 +#define CK_F_PR_SUB_8 +#define CK_F_PR_SUB_CHAR +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_SHORT +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_16 +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_64 +#define CK_F_PR_XOR_8 +#define CK_F_PR_XOR_CHAR +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_SHORT +#define CK_F_PR_XOR_UINT diff --git a/freebsd/sys/contrib/ck/include/gcc/aarch64/ck_pr.h b/freebsd/sys/contrib/ck/include/gcc/aarch64/ck_pr.h new file mode 100644 index 00000000..e739c4d5 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/gcc/aarch64/ck_pr.h @@ -0,0 +1,227 @@ +/* + * Copyright 2009-2016 Samy Al Bahra. + * Copyright 2013-2016 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_AARCH64_H +#define CK_PR_AARCH64_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include <ck_cc.h> +#include <ck_md.h> + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* + * Minimum interface requirement met. + */ +#define CK_F_PR + +CK_CC_INLINE static void +ck_pr_stall(void) +{ + + __asm__ __volatile__("" ::: "memory"); + return; +} + +#define CK_DMB_SY __asm __volatile("dmb ish" : : "r" (0) : "memory") +#define CK_DMB_LD __asm __volatile("dmb ishld" : : "r" (0) : "memory") +#define CK_DMB_ST __asm __volatile("dmb ishst" : : "r" (0) : "memory") + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + I; \ + } + +CK_PR_FENCE(atomic, CK_DMB_ST) +CK_PR_FENCE(atomic_store, CK_DMB_ST) +CK_PR_FENCE(atomic_load, CK_DMB_SY) +CK_PR_FENCE(store_atomic, CK_DMB_ST) +CK_PR_FENCE(load_atomic, CK_DMB_SY) +CK_PR_FENCE(store, CK_DMB_ST) +CK_PR_FENCE(store_load, CK_DMB_SY) +CK_PR_FENCE(load, CK_DMB_LD) +CK_PR_FENCE(load_store, CK_DMB_SY) +CK_PR_FENCE(memory, CK_DMB_SY) +CK_PR_FENCE(acquire, CK_DMB_SY) +CK_PR_FENCE(release, CK_DMB_SY) +CK_PR_FENCE(acqrel, CK_DMB_SY) +CK_PR_FENCE(lock, CK_DMB_SY) +CK_PR_FENCE(unlock, CK_DMB_SY) + +#undef CK_PR_FENCE + +#undef CK_DMB_SI +#undef CK_DMB_LD +#undef CK_DMB_ST + +#define CK_PR_LOAD(S, M, T, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + long r = 0; \ + __asm__ __volatile__(I " %w0, [%1];" \ + : "=r" (r) \ + : "r" (target) \ + : "memory"); \ + return ((T)r); \ + } +#define CK_PR_LOAD_64(S, M, T, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + long r = 0; \ + __asm__ __volatile__(I " %0, [%1];" \ + : "=r" (r) \ + : "r" (target) \ + : "memory"); \ + return ((T)r); \ + } + + +CK_PR_LOAD_64(ptr, void, void *, "ldr") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, I) +#define CK_PR_LOAD_S_64(S, T, I) CK_PR_LOAD_64(S, T, T, I) + +CK_PR_LOAD_S_64(64, uint64_t, "ldr") +CK_PR_LOAD_S(32, uint32_t, "ldr") +CK_PR_LOAD_S(16, uint16_t, "ldrh") +CK_PR_LOAD_S(8, uint8_t, "ldrb") +CK_PR_LOAD_S(uint, unsigned int, "ldr") +CK_PR_LOAD_S(int, int, "ldr") +CK_PR_LOAD_S(short, short, "ldrh") +CK_PR_LOAD_S(char, char, "ldrb") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_LOAD_S_64(double, double, "ldr") +#endif + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD_S_64 +#undef CK_PR_LOAD +#undef CK_PR_LAOD_64 + +#define CK_PR_STORE(S, M, T, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %w1, [%0]" \ + : \ + : "r" (target), \ + "r" (v) \ + : "memory"); \ + return; \ + } +#define CK_PR_STORE_64(S, M, T, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %1, [%0]" \ + : \ + : "r" (target), \ + "r" (v) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE_64(ptr, void, const void *, "str") + +#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, I) +#define CK_PR_STORE_S_64(S, T, I) CK_PR_STORE_64(S, T, T, I) + +CK_PR_STORE_S_64(64, uint64_t, "str") +CK_PR_STORE_S(32, uint32_t, "str") +CK_PR_STORE_S(16, uint16_t, "strh") +CK_PR_STORE_S(8, uint8_t, "strb") +CK_PR_STORE_S(uint, unsigned int, "str") +CK_PR_STORE_S(int, int, "str") +CK_PR_STORE_S(short, short, "strh") +CK_PR_STORE_S(char, char, "strb") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_STORE_S_64(double, double, "str") +#endif + +#undef CK_PR_STORE_S +#undef CK_PR_STORE_S_64 +#undef CK_PR_STORE +#undef CK_PR_STORE_64 + +#ifdef CK_MD_LSE_ENABLE +#include "ck_pr_lse.h" +#else +#include "ck_pr_llsc.h" +#endif + +/* + * ck_pr_neg_*() functions can only be implemented via LL/SC, as there are no + * LSE alternatives. + */ +#define CK_PR_NEG(N, M, T, W, R) \ + CK_CC_INLINE static void \ + ck_pr_neg_##N(M *target) \ + { \ + T previous = 0; \ + T tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldxr" W " %" R "0, [%2];" \ + "neg %" R "0, %" R "0;" \ + "stxr" W " %w1, %" R "0, [%2];" \ + "cbnz %w1, 1b;" \ + : "=&r" (previous), \ + "=&r" (tmp) \ + : "r" (target) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_NEG(ptr, void, void *, "", "") +CK_PR_NEG(64, uint64_t, uint64_t, "", "") + +#define CK_PR_NEG_S(S, T, W) \ + CK_PR_NEG(S, T, T, W, "w") \ + +CK_PR_NEG_S(32, uint32_t, "") +CK_PR_NEG_S(uint, unsigned int, "") +CK_PR_NEG_S(int, int, "") +CK_PR_NEG_S(16, uint16_t, "h") +CK_PR_NEG_S(8, uint8_t, "b") +CK_PR_NEG_S(short, short, "h") +CK_PR_NEG_S(char, char, "b") + +#undef CK_PR_NEG_S +#undef CK_PR_NEG + +#endif /* CK_PR_AARCH64_H */ + diff --git a/freebsd/sys/contrib/ck/include/gcc/arm/ck_f_pr.h b/freebsd/sys/contrib/ck/include/gcc/arm/ck_f_pr.h new file mode 100644 index 00000000..c508f855 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/gcc/arm/ck_f_pr.h @@ -0,0 +1,162 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_16 +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_8 +#define CK_F_PR_ADD_CHAR +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_SHORT +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_16 +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_8 +#define CK_F_PR_AND_CHAR +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_SHORT +#define CK_F_PR_AND_UINT +#define CK_F_PR_BARRIER +#define CK_F_PR_CAS_16 +#define CK_F_PR_CAS_16_VALUE +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) +#define CK_F_PR_CAS_64 +#define CK_F_PR_CAS_64_VALUE +#define CK_F_PR_CAS_DOUBLE +#define CK_F_PR_CAS_DOUBLE_VALUE +#endif +#define CK_F_PR_CAS_8 +#define CK_F_PR_CAS_8_VALUE +#define CK_F_PR_CAS_CHAR +#define CK_F_PR_CAS_CHAR_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) +#define CK_F_PR_CAS_PTR_2 +#define CK_F_PR_CAS_PTR_2_VALUE +#endif +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_SHORT +#define CK_F_PR_CAS_SHORT_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_16 +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_8 +#define CK_F_PR_DEC_CHAR +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_SHORT +#define CK_F_PR_DEC_UINT +#define CK_F_PR_FAA_16 +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_8 +#define CK_F_PR_FAA_CHAR +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_SHORT +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FAS_16 +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_8 +#define CK_F_PR_FAS_CHAR +#define CK_F_PR_FAS_INT +#define CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_SHORT +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FENCE_ATOMIC +#define CK_F_PR_FENCE_ATOMIC_LOAD +#define CK_F_PR_FENCE_ATOMIC_STORE +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_ATOMIC +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_LOAD_STORE +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STORE_ATOMIC +#define CK_F_PR_FENCE_STORE_LOAD +#define CK_F_PR_FENCE_STRICT_ATOMIC +#define CK_F_PR_FENCE_STRICT_ATOMIC_LOAD +#define CK_F_PR_FENCE_STRICT_ATOMIC_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_LOAD_ATOMIC +#define CK_F_PR_FENCE_STRICT_LOAD_STORE +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_FENCE_STRICT_STORE_ATOMIC +#define CK_F_PR_FENCE_STRICT_STORE_LOAD +#define CK_F_PR_INC_16 +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_8 +#define CK_F_PR_INC_CHAR +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_SHORT +#define CK_F_PR_INC_UINT +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_32 +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) +#define CK_F_PR_LOAD_64 +#define CK_F_PR_LOAD_DOUBLE +#endif +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_SHORT +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_NEG_16 +#define CK_F_PR_NEG_32 +#define CK_F_PR_NEG_8 +#define CK_F_PR_NEG_CHAR +#define CK_F_PR_NEG_INT +#define CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_SHORT +#define CK_F_PR_NEG_UINT +#define CK_F_PR_NOT_16 +#define CK_F_PR_NOT_32 +#define CK_F_PR_NOT_8 +#define CK_F_PR_NOT_CHAR +#define CK_F_PR_NOT_INT +#define CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_SHORT +#define CK_F_PR_NOT_UINT +#define CK_F_PR_OR_16 +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_8 +#define CK_F_PR_OR_CHAR +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_SHORT +#define CK_F_PR_OR_UINT +#define CK_F_PR_STALL +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) +#define CK_F_PR_STORE_64 +#define CK_F_PR_STORE_DOUBLE +#endif +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_SHORT +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_16 +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_8 +#define CK_F_PR_SUB_CHAR +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_SHORT +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_16 +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_8 +#define CK_F_PR_XOR_CHAR +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_SHORT +#define CK_F_PR_XOR_UINT diff --git a/freebsd/sys/contrib/ck/include/gcc/arm/ck_pr.h b/freebsd/sys/contrib/ck/include/gcc/arm/ck_pr.h new file mode 100644 index 00000000..b1f36997 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/gcc/arm/ck_pr.h @@ -0,0 +1,563 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * Copyright 2013-2015 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_ARM_H +#define CK_PR_ARM_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include <ck_cc.h> +#include <ck_md.h> + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* + * Minimum interface requirement met. + */ +#define CK_F_PR + +CK_CC_INLINE static void +ck_pr_stall(void) +{ + + __asm__ __volatile__("" ::: "memory"); + return; +} + +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) +#define CK_ISB __asm __volatile("isb" : : "r" (0) : "memory") +#define CK_DMB __asm __volatile("dmb" : : "r" (0) : "memory") +#define CK_DSB __asm __volatile("dsb" : : "r" (0) : "memory") +/* FreeBSD's toolchain doesn't accept dmb st, so use the opcode instead */ +#if defined(__FreeBSD__) && !defined(__rtems__) +#define CK_DMB_ST __asm __volatile(".word 0xf57ff05e" : : "r" (0) : "memory") +#else +#define CK_DMB_ST __asm __volatile("dmb st" : : "r" (0) : "memory") +#endif /* __FreeBSD__ */ +#else +/* armv6 doesn't have dsb/dmb/isb, and no way to wait only for stores */ +#define CK_ISB \ + __asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory") +#define CK_DSB \ + __asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory") +#define CK_DMB \ + __asm __volatile("mcr p15, 0, %0, c7, c10, 5" : : "r" (0) : "memory") +#define CK_DMB_ST CK_DMB +#endif + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + I; \ + } + +CK_PR_FENCE(atomic, CK_DMB_ST) +CK_PR_FENCE(atomic_store, CK_DMB_ST) +CK_PR_FENCE(atomic_load, CK_DMB_ST) +CK_PR_FENCE(store_atomic, CK_DMB_ST) +CK_PR_FENCE(load_atomic, CK_DMB) +CK_PR_FENCE(store, CK_DMB_ST) +CK_PR_FENCE(store_load, CK_DMB) +CK_PR_FENCE(load, CK_DMB) +CK_PR_FENCE(load_store, CK_DMB) +CK_PR_FENCE(memory, CK_DMB) +CK_PR_FENCE(acquire, CK_DMB) +CK_PR_FENCE(release, CK_DMB) +CK_PR_FENCE(acqrel, CK_DMB) +CK_PR_FENCE(lock, CK_DMB) +CK_PR_FENCE(unlock, CK_DMB) + +#undef CK_PR_FENCE + +#undef CK_ISB +#undef CK_DSB +#undef CK_DMB +#undef CK_DMB_ST + +#define CK_PR_LOAD(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + long r = 0; \ + __asm__ __volatile__(I " %0, [%1];" \ + : "=r" (r) \ + : "r" (target) \ + : "memory"); \ + return ((T)r); \ + } + +CK_PR_LOAD(ptr, void, void *, uint32_t, "ldr") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) + +CK_PR_LOAD_S(32, uint32_t, "ldr") +CK_PR_LOAD_S(16, uint16_t, "ldrh") +CK_PR_LOAD_S(8, uint8_t, "ldrb") +CK_PR_LOAD_S(uint, unsigned int, "ldr") +CK_PR_LOAD_S(int, int, "ldr") +CK_PR_LOAD_S(short, short, "ldrh") +CK_PR_LOAD_S(char, char, "ldrb") + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) + +#define CK_PR_DOUBLE_LOAD(T, N) \ +CK_CC_INLINE static T \ +ck_pr_md_load_##N(const T *target) \ +{ \ + register T ret; \ + \ + __asm __volatile("ldrexd %0, [%1]" \ + : "=&r" (ret) \ + : "r" (target) \ + : "memory", "cc"); \ + return (ret); \ +} + +CK_PR_DOUBLE_LOAD(uint64_t, 64) +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_DOUBLE_LOAD(double, double) +#endif +#undef CK_PR_DOUBLE_LOAD +#endif + +#define CK_PR_STORE(S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %1, [%0]" \ + : \ + : "r" (target), \ + "r" (v) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE(ptr, void, const void *, uint32_t, "str") + +#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) + +CK_PR_STORE_S(32, uint32_t, "str") +CK_PR_STORE_S(16, uint16_t, "strh") +CK_PR_STORE_S(8, uint8_t, "strb") +CK_PR_STORE_S(uint, unsigned int, "str") +CK_PR_STORE_S(int, int, "str") +CK_PR_STORE_S(short, short, "strh") +CK_PR_STORE_S(char, char, "strb") + +#undef CK_PR_STORE_S +#undef CK_PR_STORE + +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) + +#define CK_PR_DOUBLE_STORE(T, N) \ +CK_CC_INLINE static void \ +ck_pr_md_store_##N(const T *target, T value) \ +{ \ + T tmp; \ + uint32_t flag; \ + __asm __volatile("1: \n" \ + "ldrexd %0, [%2]\n" \ + "strexd %1, %3, [%2]\n" \ + "teq %1, #0\n" \ + "it ne \n" \ + "bne 1b\n" \ + : "=&r" (tmp), "=&r" (flag) \ + : "r" (target), "r" (value) \ + : "memory", "cc"); \ +} + +CK_PR_DOUBLE_STORE(uint64_t, 64) +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_DOUBLE_STORE(double, double) +#endif + +#undef CK_PR_DOUBLE_STORE + +#define CK_PR_DOUBLE_CAS_VALUE(T, N) \ +CK_CC_INLINE static bool \ +ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \ +{ \ + T previous; \ + int tmp; \ + \ + __asm__ __volatile__("1:" \ + "ldrexd %0, [%4];" \ + "cmp %Q0, %Q2;" \ + "ittt eq;" \ + "cmpeq %R0, %R2;" \ + "strexdeq %1, %3, [%4];" \ + "cmpeq %1, #1;" \ + "beq 1b;" \ + :"=&r" (previous), "=&r" (tmp) \ + : "r" (compare), "r" (set) , \ + "r"(target) \ + : "memory", "cc"); \ + *value = previous; \ + return (*value == compare); \ +} + +CK_PR_DOUBLE_CAS_VALUE(uint64_t, 64) +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_DOUBLE_CAS_VALUE(double, double) +#endif + +#undef CK_PR_DOUBLE_CAS_VALUE + +CK_CC_INLINE static bool +ck_pr_cas_ptr_2_value(void *target, void *compare, void *set, void *value) +{ + uint32_t *_compare = CK_CPP_CAST(uint32_t *, compare); + uint32_t *_set = CK_CPP_CAST(uint32_t *, set); + uint64_t __compare = ((uint64_t)_compare[0]) | ((uint64_t)_compare[1] << 32); + uint64_t __set = ((uint64_t)_set[0]) | ((uint64_t)_set[1] << 32); + + return (ck_pr_cas_64_value(CK_CPP_CAST(uint64_t *, target), + __compare, + __set, + CK_CPP_CAST(uint64_t *, value))); +} + +#define CK_PR_DOUBLE_CAS(T, N) \ +CK_CC_INLINE static bool \ +ck_pr_cas_##N(T *target, T compare, T set) \ +{ \ + int ret; \ + T tmp; \ + \ + __asm__ __volatile__("1:" \ + "mov %0, #0;" \ + "ldrexd %1, [%4];" \ + "cmp %Q1, %Q2;" \ + "itttt eq;" \ + "cmpeq %R1, %R2;" \ + "strexdeq %1, %3, [%4];" \ + "moveq %0, #1;" \ + "cmpeq %1, #1;" \ + "beq 1b;" \ + : "=&r" (ret), "=&r" (tmp) \ + : "r" (compare), "r" (set) , \ + "r"(target) \ + : "memory", "cc"); \ + \ + return (ret); \ +} + +CK_PR_DOUBLE_CAS(uint64_t, 64) +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_DOUBLE_CAS(double, double) +#endif + +CK_CC_INLINE static bool +ck_pr_cas_ptr_2(void *target, void *compare, void *set) +{ + uint32_t *_compare = CK_CPP_CAST(uint32_t *, compare); + uint32_t *_set = CK_CPP_CAST(uint32_t *, set); + uint64_t __compare = ((uint64_t)_compare[0]) | ((uint64_t)_compare[1] << 32); + uint64_t __set = ((uint64_t)_set[0]) | ((uint64_t)_set[1] << 32); + return (ck_pr_cas_64(CK_CPP_CAST(uint64_t *, target), + __compare, + __set)); +} + +#endif + +CK_CC_INLINE static bool +ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *value) +{ + void *previous, *tmp; + __asm__ __volatile__("1:" + "ldrex %0, [%2];" + "cmp %0, %4;" + "itt eq;" + "strexeq %1, %3, [%2];" + "cmpeq %1, #1;" + "beq 1b;" + : "=&r" (previous), + "=&r" (tmp) + : "r" (target), + "r" (set), + "r" (compare) + : "memory", "cc"); + *(void **)value = previous; + return (previous == compare); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr(void *target, void *compare, void *set) +{ + void *previous, *tmp; + __asm__ __volatile__("1:" + "ldrex %0, [%2];" + "cmp %0, %4;" + "itt eq;" + "strexeq %1, %3, [%2];" + "cmpeq %1, #1;" + "beq 1b;" + : "=&r" (previous), + "=&r" (tmp) + : "r" (target), + "r" (set), + "r" (compare) + : "memory", "cc"); + return (previous == compare); +} + +#define CK_PR_CAS(N, T, W) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \ + { \ + T previous = 0, tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldrex" W " %0, [%2];" \ + "cmp %0, %4;" \ + "itt eq;" \ + "strex" W "eq %1, %3, [%2];" \ + "cmpeq %1, #1;" \ + "beq 1b;" \ + /* \ + * Using "+&" instead of "=&" to avoid bogus \ + * clang warnings. \ + */ \ + : "+&r" (previous), \ + "+&r" (tmp) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + *value = previous; \ + return (previous == compare); \ + } \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N(T *target, T compare, T set) \ + { \ + T previous = 0, tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldrex" W " %0, [%2];" \ + "cmp %0, %4;" \ + "itt eq;" \ + "strex" W "eq %1, %3, [%2];" \ + "cmpeq %1, #1;" \ + "beq 1b;" \ + : "+&r" (previous), \ + "+&r" (tmp) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + return (previous == compare); \ + } + +CK_PR_CAS(32, uint32_t, "") +CK_PR_CAS(uint, unsigned int, "") +CK_PR_CAS(int, int, "") +CK_PR_CAS(16, uint16_t, "h") +CK_PR_CAS(8, uint8_t, "b") +CK_PR_CAS(short, short, "h") +CK_PR_CAS(char, char, "b") + + +#undef CK_PR_CAS + +#define CK_PR_FAS(N, M, T, W) \ + CK_CC_INLINE static T \ + ck_pr_fas_##N(M *target, T v) \ + { \ + T previous = 0; \ + T tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldrex" W " %0, [%2];" \ + "strex" W " %1, %3, [%2];" \ + "cmp %1, #0;" \ + "bne 1b;" \ + : "+&r" (previous), \ + "+&r" (tmp) \ + : "r" (target), \ + "r" (v) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAS(32, uint32_t, uint32_t, "") +CK_PR_FAS(ptr, void, void *, "") +CK_PR_FAS(int, int, int, "") +CK_PR_FAS(uint, unsigned int, unsigned int, "") +CK_PR_FAS(16, uint16_t, uint16_t, "h") +CK_PR_FAS(8, uint8_t, uint8_t, "b") +CK_PR_FAS(short, short, short, "h") +CK_PR_FAS(char, char, char, "b") + + +#undef CK_PR_FAS + +#define CK_PR_UNARY(O, N, M, T, I, W) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target) \ + { \ + T previous = 0; \ + T tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldrex" W " %0, [%2];" \ + I ";" \ + "strex" W " %1, %0, [%2];" \ + "cmp %1, #0;" \ + "bne 1b;" \ + : "+&r" (previous), \ + "+&r" (tmp) \ + : "r" (target) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_UNARY(inc, ptr, void, void *, "add %0, %0, #1", "") +CK_PR_UNARY(dec, ptr, void, void *, "sub %0, %0, #1", "") +CK_PR_UNARY(not, ptr, void, void *, "mvn %0, %0", "") +CK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "") + +#define CK_PR_UNARY_S(S, T, W) \ + CK_PR_UNARY(inc, S, T, T, "add %0, %0, #1", W) \ + CK_PR_UNARY(dec, S, T, T, "sub %0, %0, #1", W) \ + CK_PR_UNARY(not, S, T, T, "mvn %0, %0", W) \ + CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W) \ + +CK_PR_UNARY_S(32, uint32_t, "") +CK_PR_UNARY_S(uint, unsigned int, "") +CK_PR_UNARY_S(int, int, "") +CK_PR_UNARY_S(16, uint16_t, "h") +CK_PR_UNARY_S(8, uint8_t, "b") +CK_PR_UNARY_S(short, short, "h") +CK_PR_UNARY_S(char, char, "b") + +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY + +#define CK_PR_BINARY(O, N, M, T, I, W) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target, T delta) \ + { \ + T previous = 0; \ + T tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldrex" W " %0, [%2];" \ + I " %0, %0, %3;" \ + "strex" W " %1, %0, [%2];" \ + "cmp %1, #0;" \ + "bne 1b;" \ + : "+&r" (previous), \ + "+&r" (tmp) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "") +CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "") +CK_PR_BINARY(or, ptr, void, uintptr_t, "orr", "") +CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "") +CK_PR_BINARY(xor, ptr, void, uintptr_t, "eor", "") + +#define CK_PR_BINARY_S(S, T, W) \ + CK_PR_BINARY(and, S, T, T, "and", W) \ + CK_PR_BINARY(add, S, T, T, "add", W) \ + CK_PR_BINARY(or, S, T, T, "orr", W) \ + CK_PR_BINARY(sub, S, T, T, "sub", W) \ + CK_PR_BINARY(xor, S, T, T, "eor", W) + +CK_PR_BINARY_S(32, uint32_t, "") +CK_PR_BINARY_S(uint, unsigned int, "") +CK_PR_BINARY_S(int, int, "") +CK_PR_BINARY_S(16, uint16_t, "h") +CK_PR_BINARY_S(8, uint8_t, "b") +CK_PR_BINARY_S(short, short, "h") +CK_PR_BINARY_S(char, char, "b") + +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +CK_CC_INLINE static void * +ck_pr_faa_ptr(void *target, uintptr_t delta) +{ + uintptr_t previous, r, tmp; + + __asm__ __volatile__("1:" + "ldrex %0, [%3];" + "add %1, %4, %0;" + "strex %2, %1, [%3];" + "cmp %2, #0;" + "bne 1b;" + : "=&r" (previous), + "=&r" (r), + "=&r" (tmp) + : "r" (target), + "r" (delta) + : "memory", "cc"); + + return (void *)(previous); +} + +#define CK_PR_FAA(S, T, W) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(T *target, T delta) \ + { \ + T previous = 0, r = 0, tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldrex" W " %0, [%3];" \ + "add %1, %4, %0;" \ + "strex" W " %2, %1, [%3];" \ + "cmp %2, #0;" \ + "bne 1b;" \ + : "+&r" (previous), \ + "+&r" (r), \ + "+&r" (tmp) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAA(32, uint32_t, "") +CK_PR_FAA(uint, unsigned int, "") +CK_PR_FAA(int, int, "") +CK_PR_FAA(16, uint16_t, "h") +CK_PR_FAA(8, uint8_t, "b") +CK_PR_FAA(short, short, "h") +CK_PR_FAA(char, char, "b") + +#undef CK_PR_FAA + +#endif /* CK_PR_ARM_H */ + diff --git a/freebsd/sys/contrib/ck/include/gcc/ck_cc.h b/freebsd/sys/contrib/ck/include/gcc/ck_cc.h new file mode 100644 index 00000000..6ebc59cb --- /dev/null +++ b/freebsd/sys/contrib/ck/include/gcc/ck_cc.h @@ -0,0 +1,141 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * Copyright 2014 Paul Khuong. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_GCC_CC_H +#define CK_GCC_CC_H + +#include <ck_md.h> + +#ifdef __SUNPRO_C +#define CK_CC_UNUSED +#define CK_CC_USED +#define CK_CC_IMM +#define CK_CC_IMM_U32 +#else +#define CK_CC_UNUSED __attribute__((unused)) +#define CK_CC_USED __attribute__((used)) +#define CK_CC_IMM "i" +#if defined(__x86_64__) || defined(__x86__) +#define CK_CC_IMM_U32 "Z" +#define CK_CC_IMM_S32 "e" +#else +#define CK_CC_IMM_U32 CK_CC_IMM +#define CK_CC_IMM_S32 CK_CC_IMM +#endif /* __x86_64__ || __x86__ */ +#endif + +#ifdef __OPTIMIZE__ +#define CK_CC_INLINE CK_CC_UNUSED inline +#else +#define CK_CC_INLINE CK_CC_UNUSED +#endif + +#define CK_CC_FORCE_INLINE CK_CC_UNUSED __attribute__((always_inline)) inline +#define CK_CC_RESTRICT __restrict__ + +/* + * Packed attribute. + */ +#define CK_CC_PACKED __attribute__((packed)) + +/* + * Weak reference. + */ +#define CK_CC_WEAKREF __attribute__((weakref)) + +/* + * Alignment attribute. + */ +#define CK_CC_ALIGN(B) __attribute__((aligned(B))) + +/* + * Cache align. + */ +#define CK_CC_CACHELINE CK_CC_ALIGN(CK_MD_CACHELINE) + +/* + * These are functions which should be avoided. + */ +#ifdef __freestanding__ +#pragma GCC poison malloc free +#endif + +/* + * Branch execution hints. + */ +#define CK_CC_LIKELY(x) (__builtin_expect(!!(x), 1)) +#define CK_CC_UNLIKELY(x) (__builtin_expect(!!(x), 0)) + +/* + * Some compilers are overly strict regarding aliasing semantics. + * Unfortunately, in many cases it makes more sense to pay aliasing + * cost rather than overly expensive register spillage. + */ +#define CK_CC_ALIASED __attribute__((__may_alias__)) + +/* + * Compile-time typeof + */ +#define CK_CC_TYPEOF(X, DEFAULT) __typeof__(X) + +/* + * Portability wrappers for bitwise operations. + */ +#ifndef CK_MD_CC_BUILTIN_DISABLE +#define CK_F_CC_FFS +CK_CC_INLINE static int +ck_cc_ffs(unsigned int x) +{ + + return __builtin_ffsl(x); +} + +#define CK_F_CC_FFSL +CK_CC_INLINE static int +ck_cc_ffsl(unsigned long x) +{ + + return __builtin_ffsll(x); +} + +#define CK_F_CC_CTZ +CK_CC_INLINE static int +ck_cc_ctz(unsigned int x) +{ + + return __builtin_ctz(x); +} + +#define CK_F_CC_POPCOUNT +CK_CC_INLINE static int +ck_cc_popcount(unsigned int x) +{ + + return __builtin_popcount(x); +} +#endif /* CK_MD_CC_BUILTIN_DISABLE */ +#endif /* CK_GCC_CC_H */ diff --git a/freebsd/sys/contrib/ck/include/gcc/ck_f_pr.h b/freebsd/sys/contrib/ck/include/gcc/ck_f_pr.h new file mode 100644 index 00000000..0ef0d108 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/gcc/ck_f_pr.h @@ -0,0 +1,105 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_16 +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_64 +#define CK_F_PR_ADD_8 +#define CK_F_PR_ADD_CHAR +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_16 +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_64 +#define CK_F_PR_AND_8 +#define CK_F_PR_AND_CHAR +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_UINT +#define CK_F_PR_CAS_16 +#define CK_F_PR_CAS_16_VALUE +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_64 +#define CK_F_PR_CAS_64_VALUE +#define CK_F_PR_CAS_8 +#define CK_F_PR_CAS_8_VALUE +#define CK_F_PR_CAS_CHAR +#define CK_F_PR_CAS_CHAR_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_16 +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_64 +#define CK_F_PR_DEC_8 +#define CK_F_PR_DEC_CHAR +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_UINT +#define CK_F_PR_FAA_16 +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_64 +#define CK_F_PR_FAA_8 +#define CK_F_PR_FAA_CHAR +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_INC_16 +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_64 +#define CK_F_PR_INC_8 +#define CK_F_PR_INC_CHAR +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_UINT +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_64 +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_OR_16 +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_64 +#define CK_F_PR_OR_8 +#define CK_F_PR_OR_CHAR +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_UINT +#define CK_F_PR_STALL +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_64 +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_16 +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_64 +#define CK_F_PR_SUB_8 +#define CK_F_PR_SUB_CHAR +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_16 +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_64 +#define CK_F_PR_XOR_8 +#define CK_F_PR_XOR_CHAR +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_UINT diff --git a/freebsd/sys/contrib/ck/include/gcc/ck_pr.h b/freebsd/sys/contrib/ck/include/gcc/ck_pr.h new file mode 100644 index 00000000..108e983a --- /dev/null +++ b/freebsd/sys/contrib/ck/include/gcc/ck_pr.h @@ -0,0 +1,297 @@ +/* + * Copyright 2010 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_GCC_H +#define CK_PR_GCC_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include <ck_cc.h> + +CK_CC_INLINE static void +ck_pr_barrier(void) +{ + + __asm__ __volatile__("" ::: "memory"); + return; +} + +#ifndef CK_F_PR +#define CK_F_PR + +#include <ck_stdbool.h> +#include <ck_stdint.h> + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +#define CK_PR_ACCESS(x) (*(volatile __typeof__(x) *)&(x)) + +#define CK_PR_LOAD(S, M, T) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + T r; \ + ck_pr_barrier(); \ + r = CK_PR_ACCESS(*(const T *)target); \ + ck_pr_barrier(); \ + return (r); \ + } \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + ck_pr_barrier(); \ + CK_PR_ACCESS(*(T *)target) = v; \ + ck_pr_barrier(); \ + return; \ + } + +CK_CC_INLINE static void * +ck_pr_md_load_ptr(const void *target) +{ + void *r; + + ck_pr_barrier(); + r = CK_CC_DECONST_PTR(*(volatile void *const*)(target)); + ck_pr_barrier(); + + return r; +} + +CK_CC_INLINE static void +ck_pr_md_store_ptr(void *target, const void *v) +{ + + ck_pr_barrier(); + *(volatile void **)target = CK_CC_DECONST_PTR(v); + ck_pr_barrier(); + return; +} + +#define CK_PR_LOAD_S(S, T) CK_PR_LOAD(S, T, T) + +CK_PR_LOAD_S(char, char) +CK_PR_LOAD_S(uint, unsigned int) +CK_PR_LOAD_S(int, int) +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_LOAD_S(double, double) +#endif +CK_PR_LOAD_S(64, uint64_t) +CK_PR_LOAD_S(32, uint32_t) +CK_PR_LOAD_S(16, uint16_t) +CK_PR_LOAD_S(8, uint8_t) + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +CK_CC_INLINE static void +ck_pr_stall(void) +{ + + ck_pr_barrier(); +} + +/* + * Load and store fences are equivalent to full fences in the GCC port. + */ +#define CK_PR_FENCE(T) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __sync_synchronize(); \ + } + +CK_PR_FENCE(atomic) +CK_PR_FENCE(atomic_atomic) +CK_PR_FENCE(atomic_load) +CK_PR_FENCE(atomic_store) +CK_PR_FENCE(store_atomic) +CK_PR_FENCE(load_atomic) +CK_PR_FENCE(load) +CK_PR_FENCE(load_load) +CK_PR_FENCE(load_store) +CK_PR_FENCE(store) +CK_PR_FENCE(store_store) +CK_PR_FENCE(store_load) +CK_PR_FENCE(memory) +CK_PR_FENCE(acquire) +CK_PR_FENCE(release) +CK_PR_FENCE(acqrel) +CK_PR_FENCE(lock) +CK_PR_FENCE(unlock) + +#undef CK_PR_FENCE + +/* + * Atomic compare and swap. + */ +#define CK_PR_CAS(S, M, T) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S(M *target, T compare, T set) \ + { \ + bool z; \ + z = __sync_bool_compare_and_swap((T *)target, compare, set); \ + return z; \ + } + +CK_PR_CAS(ptr, void, void *) + +#define CK_PR_CAS_S(S, T) CK_PR_CAS(S, T, T) + +CK_PR_CAS_S(char, char) +CK_PR_CAS_S(int, int) +CK_PR_CAS_S(uint, unsigned int) +CK_PR_CAS_S(64, uint64_t) +CK_PR_CAS_S(32, uint32_t) +CK_PR_CAS_S(16, uint16_t) +CK_PR_CAS_S(8, uint8_t) + +#undef CK_PR_CAS_S +#undef CK_PR_CAS + +/* + * Compare and swap, set *v to old value of target. + */ +CK_CC_INLINE static bool +ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *v) +{ + set = __sync_val_compare_and_swap((void **)target, compare, set); + *(void **)v = set; + return (set == compare); +} + +#define CK_PR_CAS_O(S, T) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S##_value(T *target, T compare, T set, T *v) \ + { \ + set = __sync_val_compare_and_swap(target, compare, set);\ + *v = set; \ + return (set == compare); \ + } + +CK_PR_CAS_O(char, char) +CK_PR_CAS_O(int, int) +CK_PR_CAS_O(uint, unsigned int) +CK_PR_CAS_O(64, uint64_t) +CK_PR_CAS_O(32, uint32_t) +CK_PR_CAS_O(16, uint16_t) +CK_PR_CAS_O(8, uint8_t) + +#undef CK_PR_CAS_O + +/* + * Atomic fetch-and-add operations. + */ +#define CK_PR_FAA(S, M, T) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(M *target, T d) \ + { \ + d = __sync_fetch_and_add((T *)target, d); \ + return (d); \ + } + +CK_PR_FAA(ptr, void, void *) + +#define CK_PR_FAA_S(S, T) CK_PR_FAA(S, T, T) + +CK_PR_FAA_S(char, char) +CK_PR_FAA_S(uint, unsigned int) +CK_PR_FAA_S(int, int) +CK_PR_FAA_S(64, uint64_t) +CK_PR_FAA_S(32, uint32_t) +CK_PR_FAA_S(16, uint16_t) +CK_PR_FAA_S(8, uint8_t) + +#undef CK_PR_FAA_S +#undef CK_PR_FAA + +/* + * Atomic store-only binary operations. + */ +#define CK_PR_BINARY(K, S, M, T) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target, T d) \ + { \ + d = __sync_fetch_and_##K((T *)target, d); \ + return; \ + } + +#define CK_PR_BINARY_S(K, S, T) CK_PR_BINARY(K, S, T, T) + +#define CK_PR_GENERATE(K) \ + CK_PR_BINARY(K, ptr, void, void *) \ + CK_PR_BINARY_S(K, char, char) \ + CK_PR_BINARY_S(K, int, int) \ + CK_PR_BINARY_S(K, uint, unsigned int) \ + CK_PR_BINARY_S(K, 64, uint64_t) \ + CK_PR_BINARY_S(K, 32, uint32_t) \ + CK_PR_BINARY_S(K, 16, uint16_t) \ + CK_PR_BINARY_S(K, 8, uint8_t) + +CK_PR_GENERATE(add) +CK_PR_GENERATE(sub) +CK_PR_GENERATE(and) +CK_PR_GENERATE(or) +CK_PR_GENERATE(xor) + +#undef CK_PR_GENERATE +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +#define CK_PR_UNARY(S, M, T) \ + CK_CC_INLINE static void \ + ck_pr_inc_##S(M *target) \ + { \ + ck_pr_add_##S(target, (T)1); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_pr_dec_##S(M *target) \ + { \ + ck_pr_sub_##S(target, (T)1); \ + return; \ + } + +#define CK_PR_UNARY_S(S, M) CK_PR_UNARY(S, M, M) + +CK_PR_UNARY(ptr, void, void *) +CK_PR_UNARY_S(char, char) +CK_PR_UNARY_S(int, int) +CK_PR_UNARY_S(uint, unsigned int) +CK_PR_UNARY_S(64, uint64_t) +CK_PR_UNARY_S(32, uint32_t) +CK_PR_UNARY_S(16, uint16_t) +CK_PR_UNARY_S(8, uint8_t) + +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY +#endif /* !CK_F_PR */ +#endif /* CK_PR_GCC_H */ diff --git a/freebsd/sys/contrib/ck/include/gcc/ppc/ck_f_pr.h b/freebsd/sys/contrib/ck/include/gcc/ppc/ck_f_pr.h new file mode 100644 index 00000000..0aec33e4 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/gcc/ppc/ck_f_pr.h @@ -0,0 +1,79 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_UINT +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_UINT +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_INT +#define CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_LOAD_DEPENDS +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_UINT +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_SHORT +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_NEG_32 +#define CK_F_PR_NEG_INT +#define CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_UINT +#define CK_F_PR_NOT_32 +#define CK_F_PR_NOT_INT +#define CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_UINT +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_UINT +#define CK_F_PR_STALL +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_SHORT +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_UINT + diff --git a/freebsd/sys/contrib/ck/include/gcc/ppc/ck_pr.h b/freebsd/sys/contrib/ck/include/gcc/ppc/ck_pr.h new file mode 100644 index 00000000..cd7935dd --- /dev/null +++ b/freebsd/sys/contrib/ck/include/gcc/ppc/ck_pr.h @@ -0,0 +1,327 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * Copyright 2012 João Fernandes. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_PPC_H +#define CK_PR_PPC_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include <ck_cc.h> +#include <ck_md.h> + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* + * Minimum interface requirement met. + */ +#define CK_F_PR + +/* + * This bounces the hardware thread from low to medium + * priority. I am unsure of the benefits of this approach + * but it is used by the Linux kernel. + */ +CK_CC_INLINE static void +ck_pr_stall(void) +{ + + __asm__ __volatile__("or 1, 1, 1;" + "or 2, 2, 2;" ::: "memory"); + return; +} + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __asm__ __volatile__(I ::: "memory"); \ + } + +CK_PR_FENCE(atomic, "lwsync") +CK_PR_FENCE(atomic_store, "lwsync") +CK_PR_FENCE(atomic_load, "sync") +CK_PR_FENCE(store_atomic, "lwsync") +CK_PR_FENCE(load_atomic, "lwsync") +CK_PR_FENCE(store, "lwsync") +CK_PR_FENCE(store_load, "sync") +CK_PR_FENCE(load, "lwsync") +CK_PR_FENCE(load_store, "lwsync") +CK_PR_FENCE(memory, "sync") +CK_PR_FENCE(acquire, "lwsync") +CK_PR_FENCE(release, "lwsync") +CK_PR_FENCE(acqrel, "lwsync") +CK_PR_FENCE(lock, "lwsync") +CK_PR_FENCE(unlock, "lwsync") + +#undef CK_PR_FENCE + +#define CK_PR_LOAD(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + T r; \ + __asm__ __volatile__(I "%U1%X1 %0, %1" \ + : "=r" (r) \ + : "m" (*(const C *)target) \ + : "memory"); \ + return (r); \ + } + +CK_PR_LOAD(ptr, void, void *, uint32_t, "lwz") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) + +CK_PR_LOAD_S(32, uint32_t, "lwz") +CK_PR_LOAD_S(16, uint16_t, "lhz") +CK_PR_LOAD_S(8, uint8_t, "lbz") +CK_PR_LOAD_S(uint, unsigned int, "lwz") +CK_PR_LOAD_S(int, int, "lwz") +CK_PR_LOAD_S(short, short, "lhz") +CK_PR_LOAD_S(char, char, "lbz") + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +#define CK_PR_STORE(S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I "%U0%X0 %1, %0" \ + : "=m" (*(C *)target) \ + : "r" (v) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE(ptr, void, const void *, uint32_t, "stw") + +#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) + +CK_PR_STORE_S(32, uint32_t, "stw") +CK_PR_STORE_S(16, uint16_t, "sth") +CK_PR_STORE_S(8, uint8_t, "stb") +CK_PR_STORE_S(uint, unsigned int, "stw") +CK_PR_STORE_S(int, int, "stw") +CK_PR_STORE_S(short, short, "sth") +CK_PR_STORE_S(char, char, "stb") + +#undef CK_PR_STORE_S +#undef CK_PR_STORE + +#define CK_PR_CAS(N, T, M) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N##_value(M *target, T compare, T set, M *value) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "lwarx %0, 0, %1;" \ + "cmpw 0, %0, %3;" \ + "bne- 2f;" \ + "stwcx. %2, 0, %1;" \ + "bne- 1b;" \ + "2:" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + *(T *)value = previous; \ + return (previous == compare); \ + } \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N(M *target, T compare, T set) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "lwarx %0, 0, %1;" \ + "cmpw 0, %0, %3;" \ + "bne- 2f;" \ + "stwcx. %2, 0, %1;" \ + "bne- 1b;" \ + "2:" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + return (previous == compare); \ + } + +CK_PR_CAS(ptr, void *, void) +#define CK_PR_CAS_S(a, b) CK_PR_CAS(a, b, b) +CK_PR_CAS_S(32, uint32_t) +CK_PR_CAS_S(uint, unsigned int) +CK_PR_CAS_S(int, int) + +#undef CK_PR_CAS_S +#undef CK_PR_CAS + +#define CK_PR_FAS(N, M, T, W) \ + CK_CC_INLINE static T \ + ck_pr_fas_##N(M *target, T v) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %1;" \ + "st" W "cx. %2, 0, %1;" \ + "bne- 1b;" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (v) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAS(32, uint32_t, uint32_t, "w") +CK_PR_FAS(ptr, void, void *, "w") +CK_PR_FAS(int, int, int, "w") +CK_PR_FAS(uint, unsigned int, unsigned int, "w") + +#undef CK_PR_FAS + +#define CK_PR_UNARY(O, N, M, T, I, W) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %1;" \ + I ";" \ + "st" W "cx. %0, 0, %1;" \ + "bne- 1b;" \ + : "=&r" (previous) \ + : "r" (target) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_UNARY(inc, ptr, void, void *, "addic %0, %0, 1", "w") +CK_PR_UNARY(dec, ptr, void, void *, "addic %0, %0, -1", "w") +CK_PR_UNARY(not, ptr, void, void *, "not %0, %0", "w") +CK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "w") + +#define CK_PR_UNARY_S(S, T, W) \ + CK_PR_UNARY(inc, S, T, T, "addic %0, %0, 1", W) \ + CK_PR_UNARY(dec, S, T, T, "addic %0, %0, -1", W) \ + CK_PR_UNARY(not, S, T, T, "not %0, %0", W) \ + CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W) + +CK_PR_UNARY_S(32, uint32_t, "w") +CK_PR_UNARY_S(uint, unsigned int, "w") +CK_PR_UNARY_S(int, int, "w") + +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY + +#define CK_PR_BINARY(O, N, M, T, I, W) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target, T delta) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %1;" \ + I " %0, %2, %0;" \ + "st" W "cx. %0, 0, %1;" \ + "bne- 1b;" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "w") +CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "w") +CK_PR_BINARY(or, ptr, void, uintptr_t, "or", "w") +CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "w") +CK_PR_BINARY(xor, ptr, void, uintptr_t, "xor", "w") + +#define CK_PR_BINARY_S(S, T, W) \ + CK_PR_BINARY(and, S, T, T, "and", W) \ + CK_PR_BINARY(add, S, T, T, "add", W) \ + CK_PR_BINARY(or, S, T, T, "or", W) \ + CK_PR_BINARY(sub, S, T, T, "subf", W) \ + CK_PR_BINARY(xor, S, T, T, "xor", W) + +CK_PR_BINARY_S(32, uint32_t, "w") +CK_PR_BINARY_S(uint, unsigned int, "w") +CK_PR_BINARY_S(int, int, "w") + +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +CK_CC_INLINE static void * +ck_pr_faa_ptr(void *target, uintptr_t delta) +{ + uintptr_t previous, r; + + __asm__ __volatile__("1:" + "lwarx %0, 0, %2;" + "add %1, %3, %0;" + "stwcx. %1, 0, %2;" + "bne- 1b;" + : "=&r" (previous), + "=&r" (r) + : "r" (target), + "r" (delta) + : "memory", "cc"); + + return (void *)(previous); +} + +#define CK_PR_FAA(S, T, W) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(T *target, T delta) \ + { \ + T previous, r; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %2;" \ + "add %1, %3, %0;" \ + "st" W "cx. %1, 0, %2;" \ + "bne- 1b;" \ + : "=&r" (previous), \ + "=&r" (r) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAA(32, uint32_t, "w") +CK_PR_FAA(uint, unsigned int, "w") +CK_PR_FAA(int, int, "w") + +#undef CK_PR_FAA + +#endif /* CK_PR_PPC_H */ + diff --git a/freebsd/sys/contrib/ck/include/gcc/ppc64/ck_f_pr.h b/freebsd/sys/contrib/ck/include/gcc/ppc64/ck_f_pr.h new file mode 100644 index 00000000..cd54a289 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/gcc/ppc64/ck_f_pr.h @@ -0,0 +1,97 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_64 +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_64 +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_UINT +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_64 +#define CK_F_PR_CAS_64_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_64 +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_UINT +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_64 +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_64 +#define CK_F_PR_FAS_INT +#define CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FAS_DOUBLE +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_LOAD_DEPENDS +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_64 +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_UINT +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_64 +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_DOUBLE +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_SHORT +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_NEG_32 +#define CK_F_PR_NEG_64 +#define CK_F_PR_NEG_INT +#define CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_UINT +#define CK_F_PR_NOT_32 +#define CK_F_PR_NOT_64 +#define CK_F_PR_NOT_INT +#define CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_UINT +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_64 +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_UINT +#define CK_F_PR_STALL +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_64 +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_DOUBLE +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_SHORT +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_64 +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_64 +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_UINT + diff --git a/freebsd/sys/contrib/ck/include/gcc/ppc64/ck_pr.h b/freebsd/sys/contrib/ck/include/gcc/ppc64/ck_pr.h new file mode 100644 index 00000000..3f5e5db0 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/gcc/ppc64/ck_pr.h @@ -0,0 +1,427 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_PPC64_H +#define CK_PR_PPC64_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include <ck_cc.h> +#include <ck_md.h> + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* + * Minimum interface requirement met. + */ +#define CK_F_PR + +/* + * This bounces the hardware thread from low to medium + * priority. I am unsure of the benefits of this approach + * but it is used by the Linux kernel. + */ +CK_CC_INLINE static void +ck_pr_stall(void) +{ + + __asm__ __volatile__("or 1, 1, 1;" + "or 2, 2, 2;" ::: "memory"); + return; +} + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __asm__ __volatile__(I ::: "memory"); \ + } + +/* + * These are derived from: + * http://www.ibm.com/developerworks/systems/articles/powerpc.html + */ +CK_PR_FENCE(atomic, "lwsync") +CK_PR_FENCE(atomic_store, "lwsync") +CK_PR_FENCE(atomic_load, "sync") +CK_PR_FENCE(store_atomic, "lwsync") +CK_PR_FENCE(load_atomic, "lwsync") +CK_PR_FENCE(store, "lwsync") +CK_PR_FENCE(store_load, "sync") +CK_PR_FENCE(load, "lwsync") +CK_PR_FENCE(load_store, "lwsync") +CK_PR_FENCE(memory, "sync") +CK_PR_FENCE(acquire, "lwsync") +CK_PR_FENCE(release, "lwsync") +CK_PR_FENCE(acqrel, "lwsync") +CK_PR_FENCE(lock, "lwsync") +CK_PR_FENCE(unlock, "lwsync") + +#undef CK_PR_FENCE + +#define CK_PR_LOAD(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + T r; \ + __asm__ __volatile__(I "%U1%X1 %0, %1" \ + : "=r" (r) \ + : "m" (*(const C *)target) \ + : "memory"); \ + return (r); \ + } + +CK_PR_LOAD(ptr, void, void *, uint64_t, "ld") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) + +CK_PR_LOAD_S(64, uint64_t, "ld") +CK_PR_LOAD_S(32, uint32_t, "lwz") +CK_PR_LOAD_S(16, uint16_t, "lhz") +CK_PR_LOAD_S(8, uint8_t, "lbz") +CK_PR_LOAD_S(uint, unsigned int, "lwz") +CK_PR_LOAD_S(int, int, "lwz") +CK_PR_LOAD_S(short, short, "lhz") +CK_PR_LOAD_S(char, char, "lbz") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_LOAD_S(double, double, "ld") +#endif + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +#define CK_PR_STORE(S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I "%U0%X0 %1, %0" \ + : "=m" (*(C *)target) \ + : "r" (v) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE(ptr, void, const void *, uint64_t, "std") + +#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) + +CK_PR_STORE_S(64, uint64_t, "std") +CK_PR_STORE_S(32, uint32_t, "stw") +CK_PR_STORE_S(16, uint16_t, "sth") +CK_PR_STORE_S(8, uint8_t, "stb") +CK_PR_STORE_S(uint, unsigned int, "stw") +CK_PR_STORE_S(int, int, "stw") +CK_PR_STORE_S(short, short, "sth") +CK_PR_STORE_S(char, char, "stb") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_STORE_S(double, double, "std") +#endif + +#undef CK_PR_STORE_S +#undef CK_PR_STORE + +CK_CC_INLINE static bool +ck_pr_cas_64_value(uint64_t *target, uint64_t compare, uint64_t set, uint64_t *value) +{ + uint64_t previous; + + __asm__ __volatile__("1:" + "ldarx %0, 0, %1;" + "cmpd 0, %0, %3;" + "bne- 2f;" + "stdcx. %2, 0, %1;" + "bne- 1b;" + "2:" + : "=&r" (previous) + : "r" (target), + "r" (set), + "r" (compare) + : "memory", "cc"); + + *value = previous; + return (previous == compare); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *value) +{ + void *previous; + + __asm__ __volatile__("1:" + "ldarx %0, 0, %1;" + "cmpd 0, %0, %3;" + "bne- 2f;" + "stdcx. %2, 0, %1;" + "bne- 1b;" + "2:" + : "=&r" (previous) + : "r" (target), + "r" (set), + "r" (compare) + : "memory", "cc"); + + ck_pr_md_store_ptr(value, previous); + return (previous == compare); +} + +CK_CC_INLINE static bool +ck_pr_cas_64(uint64_t *target, uint64_t compare, uint64_t set) +{ + uint64_t previous; + + __asm__ __volatile__("1:" + "ldarx %0, 0, %1;" + "cmpd 0, %0, %3;" + "bne- 2f;" + "stdcx. %2, 0, %1;" + "bne- 1b;" + "2:" + : "=&r" (previous) + : "r" (target), + "r" (set), + "r" (compare) + : "memory", "cc"); + + return (previous == compare); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr(void *target, void *compare, void *set) +{ + void *previous; + + __asm__ __volatile__("1:" + "ldarx %0, 0, %1;" + "cmpd 0, %0, %3;" + "bne- 2f;" + "stdcx. %2, 0, %1;" + "bne- 1b;" + "2:" + : "=&r" (previous) + : "r" (target), + "r" (set), + "r" (compare) + : "memory", "cc"); + + return (previous == compare); +} + +#define CK_PR_CAS(N, T) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "lwarx %0, 0, %1;" \ + "cmpw 0, %0, %3;" \ + "bne- 2f;" \ + "stwcx. %2, 0, %1;" \ + "bne- 1b;" \ + "2:" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + *value = previous; \ + return (previous == compare); \ + } \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N(T *target, T compare, T set) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "lwarx %0, 0, %1;" \ + "cmpw 0, %0, %3;" \ + "bne- 2f;" \ + "stwcx. %2, 0, %1;" \ + "bne- 1b;" \ + "2:" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + return (previous == compare); \ + } + +CK_PR_CAS(32, uint32_t) +CK_PR_CAS(uint, unsigned int) +CK_PR_CAS(int, int) + +#undef CK_PR_CAS + +#define CK_PR_FAS(N, M, T, W) \ + CK_CC_INLINE static T \ + ck_pr_fas_##N(M *target, T v) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %1;" \ + "st" W "cx. %2, 0, %1;" \ + "bne- 1b;" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (v) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAS(64, uint64_t, uint64_t, "d") +CK_PR_FAS(32, uint32_t, uint32_t, "w") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_FAS(double, double, double, "d") +#endif +CK_PR_FAS(ptr, void, void *, "d") +CK_PR_FAS(int, int, int, "w") +CK_PR_FAS(uint, unsigned int, unsigned int, "w") + +#undef CK_PR_FAS + +#define CK_PR_UNARY(O, N, M, T, I, W) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %1;" \ + I ";" \ + "st" W "cx. %0, 0, %1;" \ + "bne- 1b;" \ + : "=&r" (previous) \ + : "r" (target) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_UNARY(inc, ptr, void, void *, "addic %0, %0, 1", "d") +CK_PR_UNARY(dec, ptr, void, void *, "addic %0, %0, -1", "d") +CK_PR_UNARY(not, ptr, void, void *, "not %0, %0", "d") +CK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "d") + +#define CK_PR_UNARY_S(S, T, W) \ + CK_PR_UNARY(inc, S, T, T, "addic %0, %0, 1", W) \ + CK_PR_UNARY(dec, S, T, T, "addic %0, %0, -1", W) \ + CK_PR_UNARY(not, S, T, T, "not %0, %0", W) \ + CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W) + +CK_PR_UNARY_S(64, uint64_t, "d") +CK_PR_UNARY_S(32, uint32_t, "w") +CK_PR_UNARY_S(uint, unsigned int, "w") +CK_PR_UNARY_S(int, int, "w") + +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY + +#define CK_PR_BINARY(O, N, M, T, I, W) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target, T delta) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %1;" \ + I " %0, %2, %0;" \ + "st" W "cx. %0, 0, %1;" \ + "bne- 1b;" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "d") +CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "d") +CK_PR_BINARY(or, ptr, void, uintptr_t, "or", "d") +CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "d") +CK_PR_BINARY(xor, ptr, void, uintptr_t, "xor", "d") + +#define CK_PR_BINARY_S(S, T, W) \ + CK_PR_BINARY(and, S, T, T, "and", W) \ + CK_PR_BINARY(add, S, T, T, "add", W) \ + CK_PR_BINARY(or, S, T, T, "or", W) \ + CK_PR_BINARY(sub, S, T, T, "subf", W) \ + CK_PR_BINARY(xor, S, T, T, "xor", W) + +CK_PR_BINARY_S(64, uint64_t, "d") +CK_PR_BINARY_S(32, uint32_t, "w") +CK_PR_BINARY_S(uint, unsigned int, "w") +CK_PR_BINARY_S(int, int, "w") + +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +CK_CC_INLINE static void * +ck_pr_faa_ptr(void *target, uintptr_t delta) +{ + uintptr_t previous, r; + + __asm__ __volatile__("1:" + "ldarx %0, 0, %2;" + "add %1, %3, %0;" + "stdcx. %1, 0, %2;" + "bne- 1b;" + : "=&r" (previous), + "=&r" (r) + : "r" (target), + "r" (delta) + : "memory", "cc"); + + return (void *)(previous); +} + +#define CK_PR_FAA(S, T, W) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(T *target, T delta) \ + { \ + T previous, r; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %2;" \ + "add %1, %3, %0;" \ + "st" W "cx. %1, 0, %2;" \ + "bne- 1b;" \ + : "=&r" (previous), \ + "=&r" (r) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAA(64, uint64_t, "d") +CK_PR_FAA(32, uint32_t, "w") +CK_PR_FAA(uint, unsigned int, "w") +CK_PR_FAA(int, int, "w") + +#undef CK_PR_FAA + +#endif /* CK_PR_PPC64_H */ diff --git a/freebsd/sys/contrib/ck/include/gcc/sparcv9/ck_f_pr.h b/freebsd/sys/contrib/ck/include/gcc/sparcv9/ck_f_pr.h new file mode 100644 index 00000000..0398680e --- /dev/null +++ b/freebsd/sys/contrib/ck/include/gcc/sparcv9/ck_f_pr.h @@ -0,0 +1,26 @@ +#define CK_F_PR_CAS_64 +#define CK_F_PR_CAS_64_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FAS_INT +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_STORE_64 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_DOUBLE +#define CK_F_PR_STORE_UINT +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_LOAD_64 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_DOUBLE +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR + diff --git a/freebsd/sys/contrib/ck/include/gcc/sparcv9/ck_pr.h b/freebsd/sys/contrib/ck/include/gcc/sparcv9/ck_pr.h new file mode 100644 index 00000000..7dc71725 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/gcc/sparcv9/ck_pr.h @@ -0,0 +1,228 @@ +/* + * Copyright 2009, 2010 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_SPARCV9_H +#define CK_PR_SPARCV9_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include <ck_cc.h> +#include <ck_md.h> + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* + * Minimum interface requirement met. + */ +#define CK_F_PR + +/* + * Order loads at the least. + */ +CK_CC_INLINE static void +ck_pr_stall(void) +{ + + __asm__ __volatile__("membar #LoadLoad" ::: "memory"); + return; +} + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __asm__ __volatile__(I ::: "memory"); \ + } + +/* + * Atomic operations are treated as both load and store + * operations on SPARCv9. + */ +CK_PR_FENCE(atomic, "membar #StoreStore") +CK_PR_FENCE(atomic_store, "membar #StoreStore") +CK_PR_FENCE(atomic_load, "membar #StoreLoad") +CK_PR_FENCE(store_atomic, "membar #StoreStore") +CK_PR_FENCE(load_atomic, "membar #LoadStore") +CK_PR_FENCE(store, "membar #StoreStore") +CK_PR_FENCE(store_load, "membar #StoreLoad") +CK_PR_FENCE(load, "membar #LoadLoad") +CK_PR_FENCE(load_store, "membar #LoadStore") +CK_PR_FENCE(memory, "membar #MemIssue") +CK_PR_FENCE(acquire, "membar #LoadLoad | #LoadStore") +CK_PR_FENCE(release, "membar #LoadStore | #StoreStore") +CK_PR_FENCE(acqrel, "membar #LoadLoad | #LoadStore | #StoreStore") +CK_PR_FENCE(lock, "membar #LoadLoad | #LoadStore | #StoreStore | #StoreLoad") +CK_PR_FENCE(unlock, "membar #LoadStore | #StoreStore") + +#undef CK_PR_FENCE + +#define CK_PR_LOAD(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + T r; \ + __asm__ __volatile__(I " [%1], %0" \ + : "=&r" (r) \ + : "r" (target) \ + : "memory"); \ + return (r); \ + } + +CK_PR_LOAD(ptr, void, void *, uint64_t, "ldx") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) + +CK_PR_LOAD_S(64, uint64_t, "ldx") +CK_PR_LOAD_S(32, uint32_t, "lduw") +CK_PR_LOAD_S(uint, unsigned int, "lduw") +CK_PR_LOAD_S(double, double, "ldx") +CK_PR_LOAD_S(int, int, "ldsw") + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +#define CK_PR_STORE(S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %0, [%1]" \ + : \ + : "r" (v), \ + "r" (target) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE(ptr, void, const void *, uint64_t, "stx") + +#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) + +CK_PR_STORE_S(8, uint8_t, "stub") +CK_PR_STORE_S(64, uint64_t, "stx") +CK_PR_STORE_S(32, uint32_t, "stuw") +CK_PR_STORE_S(uint, unsigned int, "stuw") +CK_PR_STORE_S(double, double, "stx") +CK_PR_STORE_S(int, int, "stsw") + +#undef CK_PR_STORE_S +#undef CK_PR_STORE + +CK_CC_INLINE static bool +ck_pr_cas_64_value(uint64_t *target, uint64_t compare, uint64_t set, uint64_t *value) +{ + + __asm__ __volatile__("casx [%1], %2, %0" + : "+&r" (set) + : "r" (target), + "r" (compare) + : "memory"); + + *value = set; + return (compare == set); +} + +CK_CC_INLINE static bool +ck_pr_cas_64(uint64_t *target, uint64_t compare, uint64_t set) +{ + + __asm__ __volatile__("casx [%1], %2, %0" + : "+&r" (set) + : "r" (target), + "r" (compare) + : "memory"); + + return (compare == set); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr(void *target, void *compare, void *set) +{ + + return ck_pr_cas_64(target, (uint64_t)compare, (uint64_t)set); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *previous) +{ + + return ck_pr_cas_64_value(target, (uint64_t)compare, (uint64_t)set, previous); +} + +#define CK_PR_CAS(N, T) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \ + { \ + __asm__ __volatile__("cas [%1], %2, %0" \ + : "+&r" (set) \ + : "r" (target), \ + "r" (compare) \ + : "memory"); \ + *value = set; \ + return (compare == set); \ + } \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N(T *target, T compare, T set) \ + { \ + __asm__ __volatile__("cas [%1], %2, %0" \ + : "+&r" (set) \ + : "r" (target), \ + "r" (compare) \ + : "memory"); \ + return (compare == set); \ + } + +CK_PR_CAS(32, uint32_t) +CK_PR_CAS(uint, unsigned int) +CK_PR_CAS(int, int) + +#undef CK_PR_CAS + +#define CK_PR_FAS(N, T) \ + CK_CC_INLINE static T \ + ck_pr_fas_##N(T *target, T update) \ + { \ + \ + __asm__ __volatile__("swap [%1], %0" \ + : "+&r" (update) \ + : "r" (target) \ + : "memory"); \ + return (update); \ + } + +CK_PR_FAS(int, int) +CK_PR_FAS(uint, unsigned int) +CK_PR_FAS(32, uint32_t) + +#undef CK_PR_FAS + +#endif /* CK_PR_SPARCV9_H */ + diff --git a/freebsd/sys/contrib/ck/include/gcc/x86/ck_f_pr.h b/freebsd/sys/contrib/ck/include/gcc/x86/ck_f_pr.h new file mode 100644 index 00000000..f82c66b0 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/gcc/x86/ck_f_pr.h @@ -0,0 +1,152 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_16 +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_8 +#define CK_F_PR_ADD_CHAR +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_16 +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_8 +#define CK_F_PR_AND_CHAR +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_UINT +#define CK_F_PR_BTC_16 +#define CK_F_PR_BTC_32 +#define CK_F_PR_BTC_INT +#define CK_F_PR_BTC_PTR +#define CK_F_PR_BTC_UINT +#define CK_F_PR_BTR_16 +#define CK_F_PR_BTR_32 +#define CK_F_PR_BTR_INT +#define CK_F_PR_BTR_PTR +#define CK_F_PR_BTR_UINT +#define CK_F_PR_BTS_16 +#define CK_F_PR_BTS_32 +#define CK_F_PR_BTS_INT +#define CK_F_PR_BTS_PTR +#define CK_F_PR_BTS_UINT +#define CK_F_PR_CAS_16 +#define CK_F_PR_CAS_16_VALUE +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_8 +#define CK_F_PR_CAS_8_VALUE +#define CK_F_PR_CAS_CHAR +#define CK_F_PR_CAS_CHAR_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_16 +#define CK_F_PR_DEC_16_ZERO +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_32_ZERO +#define CK_F_PR_DEC_8 +#define CK_F_PR_DEC_8_ZERO +#define CK_F_PR_DEC_CHAR +#define CK_F_PR_DEC_CHAR_ZERO +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_INT_ZERO +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_PTR_ZERO +#define CK_F_PR_DEC_UINT +#define CK_F_PR_DEC_UINT_ZERO +#define CK_F_PR_FAA_16 +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_8 +#define CK_F_PR_FAA_CHAR +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FAS_16 +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_8 +#define CK_F_PR_FAS_CHAR +#define CK_F_PR_FAS_INT +#define CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_LOAD_DEPENDS +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_INC_16 +#define CK_F_PR_INC_16_ZERO +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_32_ZERO +#define CK_F_PR_INC_8 +#define CK_F_PR_INC_8_ZERO +#define CK_F_PR_INC_CHAR +#define CK_F_PR_INC_CHAR_ZERO +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_INT_ZERO +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_PTR_ZERO +#define CK_F_PR_INC_UINT +#define CK_F_PR_INC_UINT_ZERO +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_NEG_16 +#define CK_F_PR_NEG_16_ZERO +#define CK_F_PR_NEG_32 +#define CK_F_PR_NEG_32_ZERO +#define CK_F_PR_NEG_8 +#define CK_F_PR_NEG_8_ZERO +#define CK_F_PR_NEG_CHAR +#define CK_F_PR_NEG_CHAR_ZERO +#define CK_F_PR_NEG_INT +#define CK_F_PR_NEG_INT_ZERO +#define CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_PTR_ZERO +#define CK_F_PR_NEG_UINT +#define CK_F_PR_NEG_UINT_ZERO +#define CK_F_PR_NOT_16 +#define CK_F_PR_NOT_32 +#define CK_F_PR_NOT_8 +#define CK_F_PR_NOT_CHAR +#define CK_F_PR_NOT_INT +#define CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_UINT +#define CK_F_PR_OR_16 +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_8 +#define CK_F_PR_OR_CHAR +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_UINT +#define CK_F_PR_STALL +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_16 +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_8 +#define CK_F_PR_SUB_CHAR +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_16 +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_8 +#define CK_F_PR_XOR_CHAR +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_UINT + diff --git a/freebsd/sys/contrib/ck/include/gcc/x86/ck_pr.h b/freebsd/sys/contrib/ck/include/gcc/x86/ck_pr.h new file mode 100644 index 00000000..3e36376f --- /dev/null +++ b/freebsd/sys/contrib/ck/include/gcc/x86/ck_pr.h @@ -0,0 +1,408 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * Copyright 2011 Devon H. O'Dell <devon.odell@gmail.com> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_X86_H +#define CK_PR_X86_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include <ck_cc.h> +#include <ck_md.h> +#include <ck_stdint.h> + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* Minimum requirements for the CK_PR interface are met. */ +#define CK_F_PR + +/* + * Prevent speculative execution in busy-wait loops (P4 <=) or "predefined + * delay". + */ +CK_CC_INLINE static void +ck_pr_stall(void) +{ + __asm__ __volatile__("pause" ::: "memory"); + return; +} + +#ifdef CK_MD_UMP +#define CK_PR_LOCK_PREFIX +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __asm__ __volatile__("" ::: "memory"); \ + return; \ + } +#else +#define CK_PR_LOCK_PREFIX "lock " +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __asm__ __volatile__(I ::: "memory"); \ + return; \ + } +#endif /* CK_MD_UMP */ + +#if defined(CK_MD_SSE_DISABLE) +/* If SSE is disabled, then use atomic operations for serialization. */ +#define CK_MD_X86_MFENCE "lock addl $0, (%%esp)" +#define CK_MD_X86_SFENCE CK_MD_X86_MFENCE +#define CK_MD_X86_LFENCE CK_MD_X86_MFENCE +#else +#define CK_MD_X86_SFENCE "sfence" +#define CK_MD_X86_LFENCE "lfence" +#define CK_MD_X86_MFENCE "mfence" +#endif /* !CK_MD_SSE_DISABLE */ + +CK_PR_FENCE(atomic, "") +CK_PR_FENCE(atomic_store, "") +CK_PR_FENCE(atomic_load, "") +CK_PR_FENCE(store_atomic, "") +CK_PR_FENCE(load_atomic, "") +CK_PR_FENCE(load, CK_MD_X86_LFENCE) +CK_PR_FENCE(load_store, CK_MD_X86_MFENCE) +CK_PR_FENCE(store, CK_MD_X86_SFENCE) +CK_PR_FENCE(store_load, CK_MD_X86_MFENCE) +CK_PR_FENCE(memory, CK_MD_X86_MFENCE) +CK_PR_FENCE(release, CK_MD_X86_MFENCE) +CK_PR_FENCE(acquire, CK_MD_X86_MFENCE) +CK_PR_FENCE(acqrel, CK_MD_X86_MFENCE) +CK_PR_FENCE(lock, CK_MD_X86_MFENCE) +CK_PR_FENCE(unlock, CK_MD_X86_MFENCE) + +#undef CK_PR_FENCE + +/* + * Atomic fetch-and-store operations. + */ +#define CK_PR_FAS(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_fas_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %0, %1" \ + : "+m" (*(C *)target), \ + "+q" (v) \ + : \ + : "memory"); \ + return v; \ + } + +CK_PR_FAS(ptr, void, void *, char, "xchgl") + +#define CK_PR_FAS_S(S, T, I) CK_PR_FAS(S, T, T, T, I) + +CK_PR_FAS_S(char, char, "xchgb") +CK_PR_FAS_S(uint, unsigned int, "xchgl") +CK_PR_FAS_S(int, int, "xchgl") +CK_PR_FAS_S(32, uint32_t, "xchgl") +CK_PR_FAS_S(16, uint16_t, "xchgw") +CK_PR_FAS_S(8, uint8_t, "xchgb") + +#undef CK_PR_FAS_S +#undef CK_PR_FAS + +#define CK_PR_LOAD(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + T r; \ + __asm__ __volatile__(I " %1, %0" \ + : "=q" (r) \ + : "m" (*(const C *)target) \ + : "memory"); \ + return (r); \ + } + +CK_PR_LOAD(ptr, void, void *, char, "movl") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) + +CK_PR_LOAD_S(char, char, "movb") +CK_PR_LOAD_S(uint, unsigned int, "movl") +CK_PR_LOAD_S(int, int, "movl") +CK_PR_LOAD_S(32, uint32_t, "movl") +CK_PR_LOAD_S(16, uint16_t, "movw") +CK_PR_LOAD_S(8, uint8_t, "movb") + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +#define CK_PR_STORE(S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %1, %0" \ + : "=m" (*(C *)target) \ + : CK_CC_IMM "q" (v) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE(ptr, void, const void *, char, "movl") + +#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) + +CK_PR_STORE_S(char, char, "movb") +CK_PR_STORE_S(uint, unsigned int, "movl") +CK_PR_STORE_S(int, int, "movl") +CK_PR_STORE_S(32, uint32_t, "movl") +CK_PR_STORE_S(16, uint16_t, "movw") +CK_PR_STORE_S(8, uint8_t, "movb") + +#undef CK_PR_STORE_S +#undef CK_PR_STORE + +/* + * Atomic fetch-and-add operations. + */ +#define CK_PR_FAA(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(M *target, T d) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %1, %0" \ + : "+m" (*(C *)target), \ + "+q" (d) \ + : \ + : "memory", "cc"); \ + return (d); \ + } + +CK_PR_FAA(ptr, void, uintptr_t, char, "xaddl") + +#define CK_PR_FAA_S(S, T, I) CK_PR_FAA(S, T, T, T, I) + +CK_PR_FAA_S(char, char, "xaddb") +CK_PR_FAA_S(uint, unsigned int, "xaddl") +CK_PR_FAA_S(int, int, "xaddl") +CK_PR_FAA_S(32, uint32_t, "xaddl") +CK_PR_FAA_S(16, uint16_t, "xaddw") +CK_PR_FAA_S(8, uint8_t, "xaddb") + +#undef CK_PR_FAA_S +#undef CK_PR_FAA + +/* + * Atomic store-only unary operations. + */ +#define CK_PR_UNARY(K, S, T, C, I) \ + CK_PR_UNARY_R(K, S, T, C, I) \ + CK_PR_UNARY_V(K, S, T, C, I) + +#define CK_PR_UNARY_R(K, S, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(T *target) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0" \ + : "+m" (*(C *)target) \ + : \ + : "memory", "cc"); \ + return; \ + } + +#define CK_PR_UNARY_V(K, S, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S##_zero(T *target, bool *r) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0; setz %1" \ + : "+m" (*(C *)target), \ + "=m" (*r) \ + : \ + : "memory", "cc"); \ + return; \ + } + + +#define CK_PR_UNARY_S(K, S, T, I) CK_PR_UNARY(K, S, T, T, I) + +#define CK_PR_GENERATE(K) \ + CK_PR_UNARY(K, ptr, void, char, #K "l") \ + CK_PR_UNARY_S(K, char, char, #K "b") \ + CK_PR_UNARY_S(K, int, int, #K "l") \ + CK_PR_UNARY_S(K, uint, unsigned int, #K "l") \ + CK_PR_UNARY_S(K, 32, uint32_t, #K "l") \ + CK_PR_UNARY_S(K, 16, uint16_t, #K "w") \ + CK_PR_UNARY_S(K, 8, uint8_t, #K "b") + +CK_PR_GENERATE(inc) +CK_PR_GENERATE(dec) +CK_PR_GENERATE(neg) + +/* not does not affect condition flags. */ +#undef CK_PR_UNARY_V +#define CK_PR_UNARY_V(a, b, c, d, e) +CK_PR_GENERATE(not) + +#undef CK_PR_GENERATE +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY_V +#undef CK_PR_UNARY_R +#undef CK_PR_UNARY + +/* + * Atomic store-only binary operations. + */ +#define CK_PR_BINARY(K, S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target, T d) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %1, %0" \ + : "+m" (*(C *)target) \ + : CK_CC_IMM "q" (d) \ + : "memory", "cc"); \ + return; \ + } + +#define CK_PR_BINARY_S(K, S, T, I) CK_PR_BINARY(K, S, T, T, T, I) + +#define CK_PR_GENERATE(K) \ + CK_PR_BINARY(K, ptr, void, uintptr_t, char, #K "l") \ + CK_PR_BINARY_S(K, char, char, #K "b") \ + CK_PR_BINARY_S(K, int, int, #K "l") \ + CK_PR_BINARY_S(K, uint, unsigned int, #K "l") \ + CK_PR_BINARY_S(K, 32, uint32_t, #K "l") \ + CK_PR_BINARY_S(K, 16, uint16_t, #K "w") \ + CK_PR_BINARY_S(K, 8, uint8_t, #K "b") + +CK_PR_GENERATE(add) +CK_PR_GENERATE(sub) +CK_PR_GENERATE(and) +CK_PR_GENERATE(or) +CK_PR_GENERATE(xor) + +#undef CK_PR_GENERATE +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +/* + * Atomic compare and swap. + */ +#define CK_PR_CAS(S, M, T, C, I) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S(M *target, T compare, T set) \ + { \ + bool z; \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %2, %0; setz %1" \ + : "+m" (*(C *)target), \ + "=a" (z) \ + : "q" (set), \ + "a" (compare) \ + : "memory", "cc"); \ + return z; \ + } + +CK_PR_CAS(ptr, void, void *, char, "cmpxchgl") + +#define CK_PR_CAS_S(S, T, I) CK_PR_CAS(S, T, T, T, I) + +CK_PR_CAS_S(char, char, "cmpxchgb") +CK_PR_CAS_S(int, int, "cmpxchgl") +CK_PR_CAS_S(uint, unsigned int, "cmpxchgl") +CK_PR_CAS_S(32, uint32_t, "cmpxchgl") +CK_PR_CAS_S(16, uint16_t, "cmpxchgw") +CK_PR_CAS_S(8, uint8_t, "cmpxchgb") + +#undef CK_PR_CAS_S +#undef CK_PR_CAS + +/* + * Compare and swap, set *v to old value of target. + */ +#define CK_PR_CAS_O(S, M, T, C, I, R) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S##_value(M *target, T compare, T set, M *v) \ + { \ + bool z; \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX "cmpxchg" I " %3, %0;" \ + "mov %% " R ", %2;" \ + "setz %1;" \ + : "+m" (*(C *)target), \ + "=a" (z), \ + "=m" (*(C *)v) \ + : "q" (set), \ + "a" (compare) \ + : "memory", "cc"); \ + return (bool)z; \ + } + +CK_PR_CAS_O(ptr, void, void *, char, "l", "eax") + +#define CK_PR_CAS_O_S(S, T, I, R) \ + CK_PR_CAS_O(S, T, T, T, I, R) + +CK_PR_CAS_O_S(char, char, "b", "al") +CK_PR_CAS_O_S(int, int, "l", "eax") +CK_PR_CAS_O_S(uint, unsigned int, "l", "eax") +CK_PR_CAS_O_S(32, uint32_t, "l", "eax") +CK_PR_CAS_O_S(16, uint16_t, "w", "ax") +CK_PR_CAS_O_S(8, uint8_t, "b", "al") + +#undef CK_PR_CAS_O_S +#undef CK_PR_CAS_O + +/* + * Atomic bit test operations. + */ +#define CK_PR_BT(K, S, T, P, C, I) \ + CK_CC_INLINE static bool \ + ck_pr_##K##_##S(T *target, unsigned int b) \ + { \ + bool c; \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I "; setc %1" \ + : "+m" (*(C *)target), \ + "=q" (c) \ + : "q" ((P)b) \ + : "memory", "cc"); \ + return (bool)c; \ + } + +#define CK_PR_BT_S(K, S, T, I) CK_PR_BT(K, S, T, T, T, I) + +#define CK_PR_GENERATE(K) \ + CK_PR_BT(K, ptr, void, uint32_t, char, #K "l %2, %0") \ + CK_PR_BT_S(K, uint, unsigned int, #K "l %2, %0") \ + CK_PR_BT_S(K, int, int, #K "l %2, %0") \ + CK_PR_BT_S(K, 32, uint32_t, #K "l %2, %0") \ + CK_PR_BT_S(K, 16, uint16_t, #K "w %w2, %0") + +CK_PR_GENERATE(btc) +CK_PR_GENERATE(bts) +CK_PR_GENERATE(btr) + +#undef CK_PR_GENERATE +#undef CK_PR_BT + +#endif /* CK_PR_X86_H */ + diff --git a/freebsd/sys/contrib/ck/include/gcc/x86_64/ck_f_pr.h b/freebsd/sys/contrib/ck/include/gcc/x86_64/ck_f_pr.h new file mode 100644 index 00000000..545f5fd6 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/gcc/x86_64/ck_f_pr.h @@ -0,0 +1,202 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_16 +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_64 +#define CK_F_PR_ADD_8 +#define CK_F_PR_ADD_CHAR +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_16 +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_64 +#define CK_F_PR_AND_8 +#define CK_F_PR_AND_CHAR +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_UINT +#define CK_F_PR_BTC_16 +#define CK_F_PR_BTC_32 +#define CK_F_PR_BTC_64 +#define CK_F_PR_BTC_INT +#define CK_F_PR_BTC_PTR +#define CK_F_PR_BTC_UINT +#define CK_F_PR_BTR_16 +#define CK_F_PR_BTR_32 +#define CK_F_PR_BTR_64 +#define CK_F_PR_BTR_INT +#define CK_F_PR_BTR_PTR +#define CK_F_PR_BTR_UINT +#define CK_F_PR_BTS_16 +#define CK_F_PR_BTS_32 +#define CK_F_PR_BTS_64 +#define CK_F_PR_BTS_INT +#define CK_F_PR_BTS_PTR +#define CK_F_PR_BTS_UINT +#define CK_F_PR_CAS_16 +#define CK_F_PR_CAS_16_8 +#define CK_F_PR_CAS_16_8_VALUE +#define CK_F_PR_CAS_16_VALUE +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_4 +#define CK_F_PR_CAS_32_4_VALUE +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_64 +#define CK_F_PR_CAS_64_2 +#define CK_F_PR_CAS_64_2_VALUE +#define CK_F_PR_CAS_64_VALUE +#define CK_F_PR_CAS_8 +#define CK_F_PR_CAS_8_16 +#define CK_F_PR_CAS_8_16_VALUE +#define CK_F_PR_CAS_8_VALUE +#define CK_F_PR_CAS_CHAR +#define CK_F_PR_CAS_CHAR_16 +#define CK_F_PR_CAS_CHAR_16_VALUE +#define CK_F_PR_CAS_CHAR_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_4 +#define CK_F_PR_CAS_INT_4_VALUE +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_2 +#define CK_F_PR_CAS_PTR_2_VALUE +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_DOUBLE +#define CK_F_PR_CAS_DOUBLE_2 +#define CK_F_PR_CAS_DOUBLE_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_4 +#define CK_F_PR_CAS_UINT_4_VALUE +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_16 +#define CK_F_PR_DEC_16_ZERO +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_32_ZERO +#define CK_F_PR_DEC_64 +#define CK_F_PR_DEC_64_ZERO +#define CK_F_PR_DEC_8 +#define CK_F_PR_DEC_8_ZERO +#define CK_F_PR_DEC_CHAR +#define CK_F_PR_DEC_CHAR_ZERO +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_INT_ZERO +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_PTR_ZERO +#define CK_F_PR_DEC_UINT +#define CK_F_PR_DEC_UINT_ZERO +#define CK_F_PR_FAA_16 +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_64 +#define CK_F_PR_FAA_8 +#define CK_F_PR_FAA_CHAR +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FAS_16 +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_64 +#define CK_F_PR_FAS_8 +#define CK_F_PR_FAS_CHAR +#define CK_F_PR_FAS_INT +#define CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FAS_DOUBLE +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_LOAD_DEPENDS +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_INC_16 +#define CK_F_PR_INC_16_ZERO +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_32_ZERO +#define CK_F_PR_INC_64 +#define CK_F_PR_INC_64_ZERO +#define CK_F_PR_INC_8 +#define CK_F_PR_INC_8_ZERO +#define CK_F_PR_INC_CHAR +#define CK_F_PR_INC_CHAR_ZERO +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_INT_ZERO +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_PTR_ZERO +#define CK_F_PR_INC_UINT +#define CK_F_PR_INC_UINT_ZERO +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_16_8 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_32_4 +#define CK_F_PR_LOAD_64 +#define CK_F_PR_LOAD_64_2 +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_8_16 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_CHAR_16 +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_INT_4 +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_PTR_2 +#define CK_F_PR_LOAD_DOUBLE +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_LOAD_UINT_4 +#define CK_F_PR_NEG_16 +#define CK_F_PR_NEG_16_ZERO +#define CK_F_PR_NEG_32 +#define CK_F_PR_NEG_32_ZERO +#define CK_F_PR_NEG_64 +#define CK_F_PR_NEG_64_ZERO +#define CK_F_PR_NEG_8 +#define CK_F_PR_NEG_8_ZERO +#define CK_F_PR_NEG_CHAR +#define CK_F_PR_NEG_CHAR_ZERO +#define CK_F_PR_NEG_INT +#define CK_F_PR_NEG_INT_ZERO +#define CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_PTR_ZERO +#define CK_F_PR_NEG_UINT +#define CK_F_PR_NEG_UINT_ZERO +#define CK_F_PR_NOT_16 +#define CK_F_PR_NOT_32 +#define CK_F_PR_NOT_64 +#define CK_F_PR_NOT_8 +#define CK_F_PR_NOT_CHAR +#define CK_F_PR_NOT_INT +#define CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_UINT +#define CK_F_PR_OR_16 +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_64 +#define CK_F_PR_OR_8 +#define CK_F_PR_OR_CHAR +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_UINT +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_64 +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_DOUBLE +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_16 +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_64 +#define CK_F_PR_SUB_8 +#define CK_F_PR_SUB_CHAR +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_16 +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_64 +#define CK_F_PR_XOR_8 +#define CK_F_PR_XOR_CHAR +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_UINT + diff --git a/freebsd/sys/contrib/ck/include/gcc/x86_64/ck_pr.h b/freebsd/sys/contrib/ck/include/gcc/x86_64/ck_pr.h new file mode 100644 index 00000000..4de13329 --- /dev/null +++ b/freebsd/sys/contrib/ck/include/gcc/x86_64/ck_pr.h @@ -0,0 +1,606 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_X86_64_H +#define CK_PR_X86_64_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include <ck_cc.h> +#include <ck_md.h> +#include <ck_stdint.h> + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* + * Support for TSX extensions. + */ +#ifdef CK_MD_RTM_ENABLE +#include "ck_pr_rtm.h" +#endif + +/* Minimum requirements for the CK_PR interface are met. */ +#define CK_F_PR + +#ifdef CK_MD_UMP +#define CK_PR_LOCK_PREFIX +#else +#define CK_PR_LOCK_PREFIX "lock " +#endif + +/* + * Prevent speculative execution in busy-wait loops (P4 <=) or "predefined + * delay". + */ +CK_CC_INLINE static void +ck_pr_stall(void) +{ + __asm__ __volatile__("pause" ::: "memory"); + return; +} + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __asm__ __volatile__(I ::: "memory"); \ + } + +/* Atomic operations are always serializing. */ +CK_PR_FENCE(atomic, "") +CK_PR_FENCE(atomic_store, "") +CK_PR_FENCE(atomic_load, "") +CK_PR_FENCE(store_atomic, "") +CK_PR_FENCE(load_atomic, "") + +/* Traditional fence interface. */ +CK_PR_FENCE(load, "lfence") +CK_PR_FENCE(load_store, "mfence") +CK_PR_FENCE(store, "sfence") +CK_PR_FENCE(store_load, "mfence") +CK_PR_FENCE(memory, "mfence") + +/* Below are stdatomic-style fences. */ + +/* + * Provides load-store and store-store ordering. However, Intel specifies that + * the WC memory model is relaxed. It is likely an sfence *is* sufficient (in + * particular, stores are not re-ordered with respect to prior loads and it is + * really just the stores that are subject to re-ordering). However, we take + * the conservative route as the manuals are too ambiguous for my taste. + */ +CK_PR_FENCE(release, "mfence") + +/* + * Provides load-load and load-store ordering. The lfence instruction ensures + * all prior load operations are complete before any subsequent instructions + * actually begin execution. However, the manual also ends up going to describe + * WC memory as a relaxed model. + */ +CK_PR_FENCE(acquire, "mfence") + +CK_PR_FENCE(acqrel, "mfence") +CK_PR_FENCE(lock, "mfence") +CK_PR_FENCE(unlock, "mfence") + +#undef CK_PR_FENCE + +/* + * Read for ownership. Older compilers will generate the 32-bit + * 3DNow! variant which is binary compatible with x86-64 variant + * of prefetchw. + */ +#ifndef CK_F_PR_RFO +#define CK_F_PR_RFO +CK_CC_INLINE static void +ck_pr_rfo(const void *m) +{ + + __asm__ __volatile__("prefetchw (%0)" + : + : "r" (m) + : "memory"); + + return; +} +#endif /* CK_F_PR_RFO */ + +/* + * Atomic fetch-and-store operations. + */ +#define CK_PR_FAS(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_fas_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %0, %1" \ + : "+m" (*(C *)target), \ + "+q" (v) \ + : \ + : "memory"); \ + return v; \ + } + +CK_PR_FAS(ptr, void, void *, char, "xchgq") + +#define CK_PR_FAS_S(S, T, I) CK_PR_FAS(S, T, T, T, I) + +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_FAS_S(double, double, "xchgq") +#endif +CK_PR_FAS_S(char, char, "xchgb") +CK_PR_FAS_S(uint, unsigned int, "xchgl") +CK_PR_FAS_S(int, int, "xchgl") +CK_PR_FAS_S(64, uint64_t, "xchgq") +CK_PR_FAS_S(32, uint32_t, "xchgl") +CK_PR_FAS_S(16, uint16_t, "xchgw") +CK_PR_FAS_S(8, uint8_t, "xchgb") + +#undef CK_PR_FAS_S +#undef CK_PR_FAS + +/* + * Atomic load-from-memory operations. + */ +#define CK_PR_LOAD(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + T r; \ + __asm__ __volatile__(I " %1, %0" \ + : "=q" (r) \ + : "m" (*(const C *)target) \ + : "memory"); \ + return (r); \ + } + +CK_PR_LOAD(ptr, void, void *, char, "movq") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) + +CK_PR_LOAD_S(char, char, "movb") +CK_PR_LOAD_S(uint, unsigned int, "movl") +CK_PR_LOAD_S(int, int, "movl") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_LOAD_S(double, double, "movq") +#endif +CK_PR_LOAD_S(64, uint64_t, "movq") +CK_PR_LOAD_S(32, uint32_t, "movl") +CK_PR_LOAD_S(16, uint16_t, "movw") +CK_PR_LOAD_S(8, uint8_t, "movb") + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +CK_CC_INLINE static void +ck_pr_load_64_2(const uint64_t target[2], uint64_t v[2]) +{ + __asm__ __volatile__("movq %%rdx, %%rcx;" + "movq %%rax, %%rbx;" + CK_PR_LOCK_PREFIX "cmpxchg16b %2;" + : "=a" (v[0]), + "=d" (v[1]) + : "m" (*(const uint64_t *)target) + : "rbx", "rcx", "memory", "cc"); + return; +} + +CK_CC_INLINE static void +ck_pr_load_ptr_2(const void *t, void *v) +{ + ck_pr_load_64_2(CK_CPP_CAST(const uint64_t *, t), + CK_CPP_CAST(uint64_t *, v)); + return; +} + +#define CK_PR_LOAD_2(S, W, T) \ + CK_CC_INLINE static void \ + ck_pr_md_load_##S##_##W(const T t[2], T v[2]) \ + { \ + ck_pr_load_64_2((const uint64_t *)(const void *)t, \ + (uint64_t *)(void *)v); \ + return; \ + } + +CK_PR_LOAD_2(char, 16, char) +CK_PR_LOAD_2(int, 4, int) +CK_PR_LOAD_2(uint, 4, unsigned int) +CK_PR_LOAD_2(32, 4, uint32_t) +CK_PR_LOAD_2(16, 8, uint16_t) +CK_PR_LOAD_2(8, 16, uint8_t) + +#undef CK_PR_LOAD_2 + +/* + * Atomic store-to-memory operations. + */ +#define CK_PR_STORE_IMM(S, M, T, C, I, K) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %1, %0" \ + : "=m" (*(C *)target) \ + : K "q" (v) \ + : "memory"); \ + return; \ + } + +#define CK_PR_STORE(S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %1, %0" \ + : "=m" (*(C *)target) \ + : "q" (v) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE_IMM(ptr, void, const void *, char, "movq", CK_CC_IMM_U32) +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_STORE(double, double, double, double, "movq") +#endif + +#define CK_PR_STORE_S(S, T, I, K) CK_PR_STORE_IMM(S, T, T, T, I, K) + +CK_PR_STORE_S(char, char, "movb", CK_CC_IMM_S32) +CK_PR_STORE_S(int, int, "movl", CK_CC_IMM_S32) +CK_PR_STORE_S(uint, unsigned int, "movl", CK_CC_IMM_U32) +CK_PR_STORE_S(64, uint64_t, "movq", CK_CC_IMM_U32) +CK_PR_STORE_S(32, uint32_t, "movl", CK_CC_IMM_U32) +CK_PR_STORE_S(16, uint16_t, "movw", CK_CC_IMM_U32) +CK_PR_STORE_S(8, uint8_t, "movb", CK_CC_IMM_U32) + +#undef CK_PR_STORE_S +#undef CK_PR_STORE_IMM +#undef CK_PR_STORE + +/* + * Atomic fetch-and-add operations. + */ +#define CK_PR_FAA(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(M *target, T d) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %1, %0" \ + : "+m" (*(C *)target), \ + "+q" (d) \ + : \ + : "memory", "cc"); \ + return (d); \ + } + +CK_PR_FAA(ptr, void, uintptr_t, char, "xaddq") + +#define CK_PR_FAA_S(S, T, I) CK_PR_FAA(S, T, T, T, I) + +CK_PR_FAA_S(char, char, "xaddb") +CK_PR_FAA_S(uint, unsigned int, "xaddl") +CK_PR_FAA_S(int, int, "xaddl") +CK_PR_FAA_S(64, uint64_t, "xaddq") +CK_PR_FAA_S(32, uint32_t, "xaddl") +CK_PR_FAA_S(16, uint16_t, "xaddw") +CK_PR_FAA_S(8, uint8_t, "xaddb") + +#undef CK_PR_FAA_S +#undef CK_PR_FAA + +/* + * Atomic store-only unary operations. + */ +#define CK_PR_UNARY(K, S, T, C, I) \ + CK_PR_UNARY_R(K, S, T, C, I) \ + CK_PR_UNARY_V(K, S, T, C, I) + +#define CK_PR_UNARY_R(K, S, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(T *target) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0" \ + : "+m" (*(C *)target) \ + : \ + : "memory", "cc"); \ + return; \ + } + +#define CK_PR_UNARY_V(K, S, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S##_zero(T *target, bool *r) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0; setz %1" \ + : "+m" (*(C *)target), \ + "=m" (*r) \ + : \ + : "memory", "cc"); \ + return; \ + } + + +#define CK_PR_UNARY_S(K, S, T, I) CK_PR_UNARY(K, S, T, T, I) + +#define CK_PR_GENERATE(K) \ + CK_PR_UNARY(K, ptr, void, char, #K "q") \ + CK_PR_UNARY_S(K, char, char, #K "b") \ + CK_PR_UNARY_S(K, int, int, #K "l") \ + CK_PR_UNARY_S(K, uint, unsigned int, #K "l") \ + CK_PR_UNARY_S(K, 64, uint64_t, #K "q") \ + CK_PR_UNARY_S(K, 32, uint32_t, #K "l") \ + CK_PR_UNARY_S(K, 16, uint16_t, #K "w") \ + CK_PR_UNARY_S(K, 8, uint8_t, #K "b") + +CK_PR_GENERATE(inc) +CK_PR_GENERATE(dec) +CK_PR_GENERATE(neg) + +/* not does not affect condition flags. */ +#undef CK_PR_UNARY_V +#define CK_PR_UNARY_V(a, b, c, d, e) +CK_PR_GENERATE(not) + +#undef CK_PR_GENERATE +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY_V +#undef CK_PR_UNARY_R +#undef CK_PR_UNARY + +/* + * Atomic store-only binary operations. + */ +#define CK_PR_BINARY(K, S, M, T, C, I, O) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target, T d) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %1, %0" \ + : "+m" (*(C *)target) \ + : O "q" (d) \ + : "memory", "cc"); \ + return; \ + } + +#define CK_PR_BINARY_S(K, S, T, I, O) CK_PR_BINARY(K, S, T, T, T, I, O) + +#define CK_PR_GENERATE(K) \ + CK_PR_BINARY(K, ptr, void, uintptr_t, char, #K "q", CK_CC_IMM_U32) \ + CK_PR_BINARY_S(K, char, char, #K "b", CK_CC_IMM_S32) \ + CK_PR_BINARY_S(K, int, int, #K "l", CK_CC_IMM_S32) \ + CK_PR_BINARY_S(K, uint, unsigned int, #K "l", CK_CC_IMM_U32) \ + CK_PR_BINARY_S(K, 64, uint64_t, #K "q", CK_CC_IMM_U32) \ + CK_PR_BINARY_S(K, 32, uint32_t, #K "l", CK_CC_IMM_U32) \ + CK_PR_BINARY_S(K, 16, uint16_t, #K "w", CK_CC_IMM_U32) \ + CK_PR_BINARY_S(K, 8, uint8_t, #K "b", CK_CC_IMM_U32) + +CK_PR_GENERATE(add) +CK_PR_GENERATE(sub) +CK_PR_GENERATE(and) +CK_PR_GENERATE(or) +CK_PR_GENERATE(xor) + +#undef CK_PR_GENERATE +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +/* + * Atomic compare and swap. + */ +#define CK_PR_CAS(S, M, T, C, I) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S(M *target, T compare, T set) \ + { \ + bool z; \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %2, %0; setz %1" \ + : "+m" (*(C *)target), \ + "=a" (z) \ + : "q" (set), \ + "a" (compare) \ + : "memory", "cc"); \ + return z; \ + } + +CK_PR_CAS(ptr, void, void *, char, "cmpxchgq") + +#define CK_PR_CAS_S(S, T, I) CK_PR_CAS(S, T, T, T, I) + +CK_PR_CAS_S(char, char, "cmpxchgb") +CK_PR_CAS_S(int, int, "cmpxchgl") +CK_PR_CAS_S(uint, unsigned int, "cmpxchgl") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_CAS_S(double, double, "cmpxchgq") +#endif +CK_PR_CAS_S(64, uint64_t, "cmpxchgq") +CK_PR_CAS_S(32, uint32_t, "cmpxchgl") +CK_PR_CAS_S(16, uint16_t, "cmpxchgw") +CK_PR_CAS_S(8, uint8_t, "cmpxchgb") + +#undef CK_PR_CAS_S +#undef CK_PR_CAS + +/* + * Compare and swap, set *v to old value of target. + */ +#define CK_PR_CAS_O(S, M, T, C, I, R) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S##_value(M *target, T compare, T set, M *v) \ + { \ + bool z; \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX "cmpxchg" I " %3, %0;" \ + "mov %% " R ", %2;" \ + "setz %1;" \ + : "+m" (*(C *)target), \ + "=a" (z), \ + "=m" (*(C *)v) \ + : "q" (set), \ + "a" (compare) \ + : "memory", "cc"); \ + return z; \ + } + +CK_PR_CAS_O(ptr, void, void *, char, "q", "rax") + +#define CK_PR_CAS_O_S(S, T, I, R) \ + CK_PR_CAS_O(S, T, T, T, I, R) + +CK_PR_CAS_O_S(char, char, "b", "al") +CK_PR_CAS_O_S(int, int, "l", "eax") +CK_PR_CAS_O_S(uint, unsigned int, "l", "eax") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_CAS_O_S(double, double, "q", "rax") +#endif +CK_PR_CAS_O_S(64, uint64_t, "q", "rax") +CK_PR_CAS_O_S(32, uint32_t, "l", "eax") +CK_PR_CAS_O_S(16, uint16_t, "w", "ax") +CK_PR_CAS_O_S(8, uint8_t, "b", "al") + +#undef CK_PR_CAS_O_S +#undef CK_PR_CAS_O + +/* + * Contrary to C-interface, alignment requirements are that of uint64_t[2]. + */ +CK_CC_INLINE static bool +ck_pr_cas_64_2(uint64_t target[2], uint64_t compare[2], uint64_t set[2]) +{ + bool z; + + __asm__ __volatile__("movq 0(%4), %%rax;" + "movq 8(%4), %%rdx;" + CK_PR_LOCK_PREFIX "cmpxchg16b %0; setz %1" + : "+m" (*target), + "=q" (z) + : "b" (set[0]), + "c" (set[1]), + "q" (compare) + : "memory", "cc", "%rax", "%rdx"); + return z; +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr_2(void *t, void *c, void *s) +{ + return ck_pr_cas_64_2(CK_CPP_CAST(uint64_t *, t), + CK_CPP_CAST(uint64_t *, c), + CK_CPP_CAST(uint64_t *, s)); +} + +CK_CC_INLINE static bool +ck_pr_cas_64_2_value(uint64_t target[2], + uint64_t compare[2], + uint64_t set[2], + uint64_t v[2]) +{ + bool z; + + __asm__ __volatile__(CK_PR_LOCK_PREFIX "cmpxchg16b %0;" + "setz %3" + : "+m" (*target), + "=a" (v[0]), + "=d" (v[1]), + "=q" (z) + : "a" (compare[0]), + "d" (compare[1]), + "b" (set[0]), + "c" (set[1]) + : "memory", "cc"); + return z; +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr_2_value(void *t, void *c, void *s, void *v) +{ + return ck_pr_cas_64_2_value(CK_CPP_CAST(uint64_t *,t), + CK_CPP_CAST(uint64_t *,c), + CK_CPP_CAST(uint64_t *,s), + CK_CPP_CAST(uint64_t *,v)); +} + +#define CK_PR_CAS_V(S, W, T) \ +CK_CC_INLINE static bool \ +ck_pr_cas_##S##_##W(T t[W], T c[W], T s[W]) \ +{ \ + return ck_pr_cas_64_2((uint64_t *)(void *)t, \ + (uint64_t *)(void *)c, \ + (uint64_t *)(void *)s); \ +} \ +CK_CC_INLINE static bool \ +ck_pr_cas_##S##_##W##_value(T *t, T c[W], T s[W], T *v) \ +{ \ + return ck_pr_cas_64_2_value((uint64_t *)(void *)t, \ + (uint64_t *)(void *)c, \ + (uint64_t *)(void *)s, \ + (uint64_t *)(void *)v); \ +} + +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_CAS_V(double, 2, double) +#endif +CK_PR_CAS_V(char, 16, char) +CK_PR_CAS_V(int, 4, int) +CK_PR_CAS_V(uint, 4, unsigned int) +CK_PR_CAS_V(32, 4, uint32_t) +CK_PR_CAS_V(16, 8, uint16_t) +CK_PR_CAS_V(8, 16, uint8_t) + +#undef CK_PR_CAS_V + +/* + * Atomic bit test operations. + */ +#define CK_PR_BT(K, S, T, P, C, I) \ + CK_CC_INLINE static bool \ + ck_pr_##K##_##S(T *target, unsigned int b) \ + { \ + bool c; \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I "; setc %1" \ + : "+m" (*(C *)target), \ + "=q" (c) \ + : "q" ((P)b) \ + : "memory", "cc"); \ + return c; \ + } + +#define CK_PR_BT_S(K, S, T, I) CK_PR_BT(K, S, T, T, T, I) + +#define CK_PR_GENERATE(K) \ + CK_PR_BT(K, ptr, void, uint64_t, char, #K "q %2, %0") \ + CK_PR_BT_S(K, uint, unsigned int, #K "l %2, %0") \ + CK_PR_BT_S(K, int, int, #K "l %2, %0") \ + CK_PR_BT_S(K, 64, uint64_t, #K "q %2, %0") \ + CK_PR_BT_S(K, 32, uint32_t, #K "l %2, %0") \ + CK_PR_BT_S(K, 16, uint16_t, #K "w %w2, %0") + +CK_PR_GENERATE(btc) +CK_PR_GENERATE(bts) +CK_PR_GENERATE(btr) + +#undef CK_PR_GENERATE +#undef CK_PR_BT + +#endif /* CK_PR_X86_64_H */ + diff --git a/freebsd/sys/contrib/ck/src/ck_epoch.c b/freebsd/sys/contrib/ck/src/ck_epoch.c new file mode 100644 index 00000000..be0f201d --- /dev/null +++ b/freebsd/sys/contrib/ck/src/ck_epoch.c @@ -0,0 +1,597 @@ +#include <machine/rtems-bsd-kernel-space.h> + +/* + * Copyright 2011-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * The implementation here is inspired from the work described in: + * Fraser, K. 2004. Practical Lock-Freedom. PhD Thesis, University + * of Cambridge Computing Laboratory. + */ + +#include <ck_backoff.h> +#include <ck_cc.h> +#include <ck_epoch.h> +#include <ck_pr.h> +#include <ck_stack.h> +#include <ck_stdbool.h> +#include <ck_string.h> + +/* + * Only three distinct values are used for reclamation, but reclamation occurs + * at e+2 rather than e+1. Any thread in a "critical section" would have + * acquired some snapshot (e) of the global epoch value (e_g) and set an active + * flag. Any hazardous references will only occur after a full memory barrier. + * For example, assume an initial e_g value of 1, e value of 0 and active value + * of 0. + * + * ck_epoch_begin(...) + * e = e_g + * active = 1 + * memory_barrier(); + * + * Any serialized reads may observe e = 0 or e = 1 with active = 0, or e = 0 or + * e = 1 with active = 1. The e_g value can only go from 1 to 2 if every thread + * has already observed the value of "1" (or the value we are incrementing + * from). This guarantees us that for any given value e_g, any threads with-in + * critical sections (referred to as "active" threads from here on) would have + * an e value of e_g-1 or e_g. This also means that hazardous references may be + * shared in both e_g-1 and e_g even if they are logically deleted in e_g. + * + * For example, assume all threads have an e value of e_g. Another thread may + * increment to e_g to e_g+1. Older threads may have a reference to an object + * which is only deleted in e_g+1. It could be that reader threads are + * executing some hash table look-ups, while some other writer thread (which + * causes epoch counter tick) actually deletes the same items that reader + * threads are looking up (this writer thread having an e value of e_g+1). + * This is possible if the writer thread re-observes the epoch after the + * counter tick. + * + * Psuedo-code for writer: + * ck_epoch_begin() + * ht_delete(x) + * ck_epoch_end() + * ck_epoch_begin() + * ht_delete(x) + * ck_epoch_end() + * + * Psuedo-code for reader: + * for (;;) { + * x = ht_lookup(x) + * ck_pr_inc(&x->value); + * } + * + * Of course, it is also possible for references logically deleted at e_g-1 to + * still be accessed at e_g as threads are "active" at the same time + * (real-world time) mutating shared objects. + * + * Now, if the epoch counter is ticked to e_g+1, then no new hazardous + * references could exist to objects logically deleted at e_g-1. The reason for + * this is that at e_g+1, all epoch read-side critical sections started at + * e_g-1 must have been completed. If any epoch read-side critical sections at + * e_g-1 were still active, then we would never increment to e_g+1 (active != 0 + * ^ e != e_g). Additionally, e_g may still have hazardous references to + * objects logically deleted at e_g-1 which means objects logically deleted at + * e_g-1 cannot be deleted at e_g+1 unless all threads have observed e_g+1 + * (since it is valid for active threads to be at e_g and threads at e_g still + * require safe memory accesses). + * + * However, at e_g+2, all active threads must be either at e_g+1 or e_g+2. + * Though e_g+2 may share hazardous references with e_g+1, and e_g+1 shares + * hazardous references to e_g, no active threads are at e_g or e_g-1. This + * means no hazardous references could exist to objects deleted at e_g-1 (at + * e_g+2). + * + * To summarize these important points, + * 1) Active threads will always have a value of e_g or e_g-1. + * 2) Items that are logically deleted e_g or e_g-1 cannot be physically + * deleted. + * 3) Objects logically deleted at e_g-1 can be physically destroyed at e_g+2 + * or at e_g+1 if no threads are at e_g. + * + * Last but not least, if we are at e_g+2, then no active thread is at e_g + * which means it is safe to apply modulo-3 arithmetic to e_g value in order to + * re-use e_g to represent the e_g+3 state. This means it is sufficient to + * represent e_g using only the values 0, 1 or 2. Every time a thread re-visits + * a e_g (which can be determined with a non-empty deferral list) it can assume + * objects in the e_g deferral list involved at least three e_g transitions and + * are thus, safe, for physical deletion. + * + * Blocking semantics for epoch reclamation have additional restrictions. + * Though we only require three deferral lists, reasonable blocking semantics + * must be able to more gracefully handle bursty write work-loads which could + * easily cause e_g wrap-around if modulo-3 arithmetic is used. This allows for + * easy-to-trigger live-lock situations. The work-around to this is to not + * apply modulo arithmetic to e_g but only to deferral list indexing. + */ +#define CK_EPOCH_GRACE 3U + +enum { + CK_EPOCH_STATE_USED = 0, + CK_EPOCH_STATE_FREE = 1 +}; + +CK_STACK_CONTAINER(struct ck_epoch_record, record_next, + ck_epoch_record_container) +CK_STACK_CONTAINER(struct ck_epoch_entry, stack_entry, + ck_epoch_entry_container) + +#define CK_EPOCH_SENSE_MASK (CK_EPOCH_SENSE - 1) + +bool +_ck_epoch_delref(struct ck_epoch_record *record, + struct ck_epoch_section *section) +{ + struct ck_epoch_ref *current, *other; + unsigned int i = section->bucket; + + current = &record->local.bucket[i]; + current->count--; + + if (current->count > 0) + return false; + + /* + * If the current bucket no longer has any references, then + * determine whether we have already transitioned into a newer + * epoch. If so, then make sure to update our shared snapshot + * to allow for forward progress. + * + * If no other active bucket exists, then the record will go + * inactive in order to allow for forward progress. + */ + other = &record->local.bucket[(i + 1) & CK_EPOCH_SENSE_MASK]; + if (other->count > 0 && + ((int)(current->epoch - other->epoch) < 0)) { + /* + * The other epoch value is actually the newest, + * transition to it. + */ + ck_pr_store_uint(&record->epoch, other->epoch); + } + + return true; +} + +void +_ck_epoch_addref(struct ck_epoch_record *record, + struct ck_epoch_section *section) +{ + struct ck_epoch *global = record->global; + struct ck_epoch_ref *ref; + unsigned int epoch, i; + + epoch = ck_pr_load_uint(&global->epoch); + i = epoch & CK_EPOCH_SENSE_MASK; + ref = &record->local.bucket[i]; + + if (ref->count++ == 0) { +#ifndef CK_MD_TSO + struct ck_epoch_ref *previous; + + /* + * The system has already ticked. If another non-zero bucket + * exists, make sure to order our observations with respect + * to it. Otherwise, it is possible to acquire a reference + * from the previous epoch generation. + * + * On TSO architectures, the monoticity of the global counter + * and load-{store, load} ordering are sufficient to guarantee + * this ordering. + */ + previous = &record->local.bucket[(i + 1) & + CK_EPOCH_SENSE_MASK]; + if (previous->count > 0) + ck_pr_fence_acqrel(); +#endif /* !CK_MD_TSO */ + + /* + * If this is this is a new reference into the current + * bucket then cache the associated epoch value. + */ + ref->epoch = epoch; + } + + section->bucket = i; + return; +} + +void +ck_epoch_init(struct ck_epoch *global) +{ + + ck_stack_init(&global->records); + global->epoch = 1; + global->n_free = 0; + ck_pr_fence_store(); + return; +} + +struct ck_epoch_record * +ck_epoch_recycle(struct ck_epoch *global, void *ct) +{ + struct ck_epoch_record *record; + ck_stack_entry_t *cursor; + unsigned int state; + + if (ck_pr_load_uint(&global->n_free) == 0) + return NULL; + + CK_STACK_FOREACH(&global->records, cursor) { + record = ck_epoch_record_container(cursor); + + if (ck_pr_load_uint(&record->state) == CK_EPOCH_STATE_FREE) { + /* Serialize with respect to deferral list clean-up. */ + ck_pr_fence_load(); + state = ck_pr_fas_uint(&record->state, + CK_EPOCH_STATE_USED); + if (state == CK_EPOCH_STATE_FREE) { + ck_pr_dec_uint(&global->n_free); + ck_pr_store_ptr(&record->ct, ct); + + /* + * The context pointer is ordered by a + * subsequent protected section. + */ + return record; + } + } + } + + return NULL; +} + +void +ck_epoch_register(struct ck_epoch *global, struct ck_epoch_record *record, + void *ct) +{ + size_t i; + + record->global = global; + record->state = CK_EPOCH_STATE_USED; + record->active = 0; + record->epoch = 0; + record->n_dispatch = 0; + record->n_peak = 0; + record->n_pending = 0; + record->ct = ct; + memset(&record->local, 0, sizeof record->local); + + for (i = 0; i < CK_EPOCH_LENGTH; i++) + ck_stack_init(&record->pending[i]); + + ck_pr_fence_store(); + ck_stack_push_upmc(&global->records, &record->record_next); + return; +} + +void +ck_epoch_unregister(struct ck_epoch_record *record) +{ + struct ck_epoch *global = record->global; + size_t i; + + record->active = 0; + record->epoch = 0; + record->n_dispatch = 0; + record->n_peak = 0; + record->n_pending = 0; + memset(&record->local, 0, sizeof record->local); + + for (i = 0; i < CK_EPOCH_LENGTH; i++) + ck_stack_init(&record->pending[i]); + + ck_pr_store_ptr(&record->ct, NULL); + ck_pr_fence_store(); + ck_pr_store_uint(&record->state, CK_EPOCH_STATE_FREE); + ck_pr_inc_uint(&global->n_free); + return; +} + +static struct ck_epoch_record * +ck_epoch_scan(struct ck_epoch *global, + struct ck_epoch_record *cr, + unsigned int epoch, + bool *af) +{ + ck_stack_entry_t *cursor; + + if (cr == NULL) { + cursor = CK_STACK_FIRST(&global->records); + *af = false; + } else { + cursor = &cr->record_next; + *af = true; + } + + while (cursor != NULL) { + unsigned int state, active; + + cr = ck_epoch_record_container(cursor); + + state = ck_pr_load_uint(&cr->state); + if (state & CK_EPOCH_STATE_FREE) { + cursor = CK_STACK_NEXT(cursor); + continue; + } + + active = ck_pr_load_uint(&cr->active); + *af |= active; + + if (active != 0 && ck_pr_load_uint(&cr->epoch) != epoch) + return cr; + + cursor = CK_STACK_NEXT(cursor); + } + + return NULL; +} + +static void +ck_epoch_dispatch(struct ck_epoch_record *record, unsigned int e, ck_stack_t *deferred) +{ + unsigned int epoch = e & (CK_EPOCH_LENGTH - 1); + ck_stack_entry_t *head, *next, *cursor; + unsigned int n_pending, n_peak; + unsigned int i = 0; + + head = ck_stack_batch_pop_upmc(&record->pending[epoch]); + for (cursor = head; cursor != NULL; cursor = next) { + struct ck_epoch_entry *entry = + ck_epoch_entry_container(cursor); + + next = CK_STACK_NEXT(cursor); + if (deferred != NULL) + ck_stack_push_spnc(deferred, &entry->stack_entry); + else + entry->function(entry); + i++; + } + + n_peak = ck_pr_load_uint(&record->n_peak); + n_pending = ck_pr_load_uint(&record->n_pending); + + /* We don't require accuracy around peak calculation. */ + if (n_pending > n_peak) + ck_pr_store_uint(&record->n_peak, n_peak); + + if (i > 0) { + ck_pr_add_uint(&record->n_dispatch, i); + ck_pr_sub_uint(&record->n_pending, i); + } + + return; +} + +/* + * Reclaim all objects associated with a record. + */ +void +ck_epoch_reclaim(struct ck_epoch_record *record) +{ + unsigned int epoch; + + for (epoch = 0; epoch < CK_EPOCH_LENGTH; epoch++) + ck_epoch_dispatch(record, epoch, NULL); + + return; +} + +CK_CC_FORCE_INLINE static void +epoch_block(struct ck_epoch *global, struct ck_epoch_record *cr, + ck_epoch_wait_cb_t *cb, void *ct) +{ + + if (cb != NULL) + cb(global, cr, ct); + + return; +} + +/* + * This function must not be called with-in read section. + */ +void +ck_epoch_synchronize_wait(struct ck_epoch *global, + ck_epoch_wait_cb_t *cb, void *ct) +{ + struct ck_epoch_record *cr; + unsigned int delta, epoch, goal, i; + bool active; + + ck_pr_fence_memory(); + + /* + * The observation of the global epoch must be ordered with respect to + * all prior operations. The re-ordering of loads is permitted given + * monoticity of global epoch counter. + * + * If UINT_MAX concurrent mutations were to occur then it is possible + * to encounter an ABA-issue. If this is a concern, consider tuning + * write-side concurrency. + */ + delta = epoch = ck_pr_load_uint(&global->epoch); + goal = epoch + CK_EPOCH_GRACE; + + for (i = 0, cr = NULL; i < CK_EPOCH_GRACE - 1; cr = NULL, i++) { + bool r; + + /* + * Determine whether all threads have observed the current + * epoch with respect to the updates on invocation. + */ + while (cr = ck_epoch_scan(global, cr, delta, &active), + cr != NULL) { + unsigned int e_d; + + ck_pr_stall(); + + /* + * Another writer may have already observed a grace + * period. + */ + e_d = ck_pr_load_uint(&global->epoch); + if (e_d == delta) { + epoch_block(global, cr, cb, ct); + continue; + } + + /* + * If the epoch has been updated, we may have already + * met our goal. + */ + delta = e_d; + if ((goal > epoch) & (delta >= goal)) + goto leave; + + epoch_block(global, cr, cb, ct); + + /* + * If the epoch has been updated, then a grace period + * requires that all threads are observed idle at the + * same epoch. + */ + cr = NULL; + } + + /* + * If we have observed all threads as inactive, then we assume + * we are at a grace period. + */ + if (active == false) + break; + + /* + * Increment current epoch. CAS semantics are used to eliminate + * increment operations for synchronization that occurs for the + * same global epoch value snapshot. + * + * If we can guarantee there will only be one active barrier or + * epoch tick at a given time, then it is sufficient to use an + * increment operation. In a multi-barrier workload, however, + * it is possible to overflow the epoch value if we apply + * modulo-3 arithmetic. + */ + r = ck_pr_cas_uint_value(&global->epoch, delta, delta + 1, + &delta); + + /* Order subsequent thread active checks. */ + ck_pr_fence_atomic_load(); + + /* + * If CAS has succeeded, then set delta to latest snapshot. + * Otherwise, we have just acquired latest snapshot. + */ + delta = delta + r; + } + + /* + * A majority of use-cases will not require full barrier semantics. + * However, if non-temporal instructions are used, full barrier + * semantics are necessary. + */ +leave: + ck_pr_fence_memory(); + return; +} + +void +ck_epoch_synchronize(struct ck_epoch_record *record) +{ + + ck_epoch_synchronize_wait(record->global, NULL, NULL); + return; +} + +void +ck_epoch_barrier(struct ck_epoch_record *record) +{ + + ck_epoch_synchronize(record); + ck_epoch_reclaim(record); + return; +} + +void +ck_epoch_barrier_wait(struct ck_epoch_record *record, ck_epoch_wait_cb_t *cb, + void *ct) +{ + + ck_epoch_synchronize_wait(record->global, cb, ct); + ck_epoch_reclaim(record); + return; +} + +/* + * It may be worth it to actually apply these deferral semantics to an epoch + * that was observed at ck_epoch_call time. The problem is that the latter + * would require a full fence. + * + * ck_epoch_call will dispatch to the latest epoch snapshot that was observed. + * There are cases where it will fail to reclaim as early as it could. If this + * becomes a problem, we could actually use a heap for epoch buckets but that + * is far from ideal too. + */ +bool +ck_epoch_poll_deferred(struct ck_epoch_record *record, ck_stack_t *deferred) +{ + bool active; + unsigned int epoch; + struct ck_epoch_record *cr = NULL; + struct ck_epoch *global = record->global; + + epoch = ck_pr_load_uint(&global->epoch); + + /* Serialize epoch snapshots with respect to global epoch. */ + ck_pr_fence_memory(); + cr = ck_epoch_scan(global, cr, epoch, &active); + if (cr != NULL) { + record->epoch = epoch; + return false; + } + + /* We are at a grace period if all threads are inactive. */ + if (active == false) { + record->epoch = epoch; + for (epoch = 0; epoch < CK_EPOCH_LENGTH; epoch++) + ck_epoch_dispatch(record, epoch, deferred); + + return true; + } + + /* If an active thread exists, rely on epoch observation. */ + (void)ck_pr_cas_uint(&global->epoch, epoch, epoch + 1); + + ck_epoch_dispatch(record, epoch + 1, deferred); + return true; +} + +bool +ck_epoch_poll(struct ck_epoch_record *record) +{ + + return ck_epoch_poll_deferred(record, NULL); +} diff --git a/freebsd/sys/dev/bce/if_bce.c b/freebsd/sys/dev/bce/if_bce.c index a2e4c804..61c9708d 100644 --- a/freebsd/sys/dev/bce/if_bce.c +++ b/freebsd/sys/dev/bce/if_bce.c @@ -531,7 +531,8 @@ MODULE_DEPEND(bce, miibus, 1, 1, 1); DRIVER_MODULE(bce, pci, bce_driver, bce_devclass, NULL, NULL); DRIVER_MODULE(miibus, bce, miibus_driver, miibus_devclass, NULL, NULL); - +MODULE_PNP_INFO("U16:vendor;U16:device;U16:#;U16:#;D:#", pci, bce, + bce_devs, sizeof(bce_devs[0]), nitems(bce_devs) - 1); /****************************************************************************/ /* Tunable device values */ @@ -8118,7 +8119,7 @@ bce_set_rx_mode(struct bce_softc *sc) DBPRINT(sc, BCE_INFO_MISC, "Enabling selective multicast mode.\n"); if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; h = ether_crc32_le(LLADDR((struct sockaddr_dl *) diff --git a/freebsd/sys/dev/bfe/if_bfe.c b/freebsd/sys/dev/bfe/if_bfe.c index 63825a99..c07d87fb 100644 --- a/freebsd/sys/dev/bfe/if_bfe.c +++ b/freebsd/sys/dev/bfe/if_bfe.c @@ -1111,7 +1111,7 @@ bfe_set_rx_mode(struct bfe_softc *sc) else { val &= ~BFE_RXCONF_ALLMULTI; if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; bfe_cam_write(sc, diff --git a/freebsd/sys/dev/bge/if_bge.c b/freebsd/sys/dev/bge/if_bge.c index 956ee52f..aba0b05d 100644 --- a/freebsd/sys/dev/bge/if_bge.c +++ b/freebsd/sys/dev/bge/if_bge.c @@ -102,6 +102,7 @@ __FBSDID("$FreeBSD$"); #include <netinet/in.h> #include <netinet/ip.h> #include <netinet/tcp.h> +#include <netinet/netdump/netdump.h> #include <machine/bus.h> #include <machine/resource.h> @@ -428,8 +429,9 @@ static int bge_encap(struct bge_softc *, struct mbuf **, uint32_t *); static void bge_intr(void *); static int bge_msi_intr(void *); static void bge_intr_task(void *, int); -static void bge_start_locked(if_t); static void bge_start(if_t); +static void bge_start_locked(if_t); +static void bge_start_tx(struct bge_softc *, uint32_t); static int bge_ioctl(if_t, u_long, caddr_t); static void bge_init_locked(struct bge_softc *); static void bge_init(void *); @@ -519,6 +521,8 @@ static void bge_add_sysctl_stats(struct bge_softc *, struct sysctl_ctx_list *, struct sysctl_oid_list *); static int bge_sysctl_stats(SYSCTL_HANDLER_ARGS); +NETDUMP_DEFINE(bge); + static device_method_t bge_methods[] = { /* Device interface */ DEVMETHOD(device_probe, bge_probe), @@ -3943,8 +3947,12 @@ again: if (error) { ether_ifdetach(ifp); device_printf(sc->bge_dev, "couldn't set up irq\n"); + goto fail; } + /* Attach driver netdump methods. */ + NETDUMP_SET(ifp, bge); + fail: if (error) bge_detach(dev); @@ -5391,22 +5399,26 @@ bge_start_locked(if_t ifp) if_bpfmtap(ifp, m_head); } - if (count > 0) { - bus_dmamap_sync(sc->bge_cdata.bge_tx_ring_tag, - sc->bge_cdata.bge_tx_ring_map, BUS_DMASYNC_PREWRITE); - /* Transmit. */ + if (count > 0) + bge_start_tx(sc, prodidx); +} + +static void +bge_start_tx(struct bge_softc *sc, uint32_t prodidx) +{ + + bus_dmamap_sync(sc->bge_cdata.bge_tx_ring_tag, + sc->bge_cdata.bge_tx_ring_map, BUS_DMASYNC_PREWRITE); + /* Transmit. */ + bge_writembx(sc, BGE_MBX_TX_HOST_PROD0_LO, prodidx); + /* 5700 b2 errata */ + if (sc->bge_chiprev == BGE_CHIPREV_5700_BX) bge_writembx(sc, BGE_MBX_TX_HOST_PROD0_LO, prodidx); - /* 5700 b2 errata */ - if (sc->bge_chiprev == BGE_CHIPREV_5700_BX) - bge_writembx(sc, BGE_MBX_TX_HOST_PROD0_LO, prodidx); - sc->bge_tx_prodidx = prodidx; + sc->bge_tx_prodidx = prodidx; - /* - * Set a timeout in case the chip goes out to lunch. - */ - sc->bge_timer = BGE_TX_TIMEOUT; - } + /* Set a timeout in case the chip goes out to lunch. */ + sc->bge_timer = BGE_TX_TIMEOUT; } /* @@ -6798,3 +6810,74 @@ bge_get_counter(if_t ifp, ift_counter cnt) return (if_get_counter_default(ifp, cnt)); } } + +#ifdef NETDUMP +static void +bge_netdump_init(if_t ifp, int *nrxr, int *ncl, int *clsize) +{ + struct bge_softc *sc; + + sc = if_getsoftc(ifp); + BGE_LOCK(sc); + *nrxr = sc->bge_return_ring_cnt; + *ncl = NETDUMP_MAX_IN_FLIGHT; + if ((sc->bge_flags & BGE_FLAG_JUMBO_STD) != 0 && + (if_getmtu(sc->bge_ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN + + ETHER_VLAN_ENCAP_LEN > (MCLBYTES - ETHER_ALIGN))) + *clsize = MJUM9BYTES; + else + *clsize = MCLBYTES; + BGE_UNLOCK(sc); +} + +static void +bge_netdump_event(if_t ifp __unused, enum netdump_ev event __unused) +{ +} + +static int +bge_netdump_transmit(if_t ifp, struct mbuf *m) +{ + struct bge_softc *sc; + uint32_t prodidx; + int error; + + sc = if_getsoftc(ifp); + if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != + IFF_DRV_RUNNING) + return (1); + + prodidx = sc->bge_tx_prodidx; + error = bge_encap(sc, &m, &prodidx); + if (error == 0) + bge_start_tx(sc, prodidx); + return (error); +} + +static int +bge_netdump_poll(if_t ifp, int count) +{ + struct bge_softc *sc; + uint32_t rx_prod, tx_cons; + + sc = if_getsoftc(ifp); + if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != + IFF_DRV_RUNNING) + return (1); + + bus_dmamap_sync(sc->bge_cdata.bge_status_tag, + sc->bge_cdata.bge_status_map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + + rx_prod = sc->bge_ldata.bge_status_block->bge_idx[0].bge_rx_prod_idx; + tx_cons = sc->bge_ldata.bge_status_block->bge_idx[0].bge_tx_cons_idx; + + bus_dmamap_sync(sc->bge_cdata.bge_status_tag, + sc->bge_cdata.bge_status_map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + + (void)bge_rxeof(sc, rx_prod, 0); + bge_txeof(sc, tx_cons); + return (0); +} +#endif /* NETDUMP */ diff --git a/freebsd/sys/dev/dc/if_dc.c b/freebsd/sys/dev/dc/if_dc.c index 3339b83d..ac34a20a 100644 --- a/freebsd/sys/dev/dc/if_dc.c +++ b/freebsd/sys/dev/dc/if_dc.c @@ -999,7 +999,7 @@ dc_setfilt_21143(struct dc_softc *sc) DC_CLRBIT(sc, DC_NETCFG, DC_NETCFG_RX_ALLMULTI); if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; h = dc_mchash_le(sc, @@ -1077,7 +1077,7 @@ dc_setfilt_admtek(struct dc_softc *sc) /* Now program new ones. */ if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; if (DC_IS_CENTAUR(sc)) @@ -1150,7 +1150,7 @@ dc_setfilt_asix(struct dc_softc *sc) /* now program new ones */ if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; h = dc_mchash_be(LLADDR((struct sockaddr_dl *)ifma->ifma_addr)); @@ -1211,7 +1211,7 @@ dc_setfilt_uli(struct dc_softc *sc) /* Now build perfect filters. */ mcnt = 0; if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; if (mcnt >= DC_ULI_FILTER_NPERF) { @@ -1296,7 +1296,7 @@ dc_setfilt_xircom(struct dc_softc *sc) DC_CLRBIT(sc, DC_NETCFG, DC_NETCFG_RX_ALLMULTI); if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; h = dc_mchash_le(sc, diff --git a/freebsd/sys/dev/dwc/if_dwc.c b/freebsd/sys/dev/dwc/if_dwc.c index badbd807..895fdfe5 100644 --- a/freebsd/sys/dev/dwc/if_dwc.c +++ b/freebsd/sys/dev/dwc/if_dwc.c @@ -702,7 +702,7 @@ dwc_setup_rxfilter(struct dwc_softc *sc) for (i = 0; i < nhash; i++) hash[i] = 0; if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &sc->ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &sc->ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; crc = ether_crc32_le(LLADDR((struct sockaddr_dl *) diff --git a/freebsd/sys/dev/e1000/e1000_80003es2lan.c b/freebsd/sys/dev/e1000/e1000_80003es2lan.c index 50175d2a..6b05840f 100644 --- a/freebsd/sys/dev/e1000/e1000_80003es2lan.c +++ b/freebsd/sys/dev/e1000/e1000_80003es2lan.c @@ -62,7 +62,6 @@ static s32 e1000_reset_hw_80003es2lan(struct e1000_hw *hw); static s32 e1000_init_hw_80003es2lan(struct e1000_hw *hw); static s32 e1000_setup_copper_link_80003es2lan(struct e1000_hw *hw); static void e1000_clear_hw_cntrs_80003es2lan(struct e1000_hw *hw); -static s32 e1000_acquire_swfw_sync_80003es2lan(struct e1000_hw *hw, u16 mask); static s32 e1000_cfg_kmrn_10_100_80003es2lan(struct e1000_hw *hw, u16 duplex); static s32 e1000_cfg_kmrn_1000_80003es2lan(struct e1000_hw *hw); static s32 e1000_cfg_on_link_up_80003es2lan(struct e1000_hw *hw); @@ -71,7 +70,6 @@ static s32 e1000_read_kmrn_reg_80003es2lan(struct e1000_hw *hw, u32 offset, static s32 e1000_write_kmrn_reg_80003es2lan(struct e1000_hw *hw, u32 offset, u16 data); static void e1000_initialize_hw_bits_80003es2lan(struct e1000_hw *hw); -static void e1000_release_swfw_sync_80003es2lan(struct e1000_hw *hw, u16 mask); static s32 e1000_read_mac_addr_80003es2lan(struct e1000_hw *hw); static void e1000_power_down_phy_copper_80003es2lan(struct e1000_hw *hw); @@ -302,7 +300,7 @@ static s32 e1000_acquire_phy_80003es2lan(struct e1000_hw *hw) DEBUGFUNC("e1000_acquire_phy_80003es2lan"); mask = hw->bus.func ? E1000_SWFW_PHY1_SM : E1000_SWFW_PHY0_SM; - return e1000_acquire_swfw_sync_80003es2lan(hw, mask); + return e1000_acquire_swfw_sync(hw, mask); } /** @@ -318,7 +316,7 @@ static void e1000_release_phy_80003es2lan(struct e1000_hw *hw) DEBUGFUNC("e1000_release_phy_80003es2lan"); mask = hw->bus.func ? E1000_SWFW_PHY1_SM : E1000_SWFW_PHY0_SM; - e1000_release_swfw_sync_80003es2lan(hw, mask); + e1000_release_swfw_sync(hw, mask); } /** @@ -336,7 +334,7 @@ static s32 e1000_acquire_mac_csr_80003es2lan(struct e1000_hw *hw) mask = E1000_SWFW_CSR_SM; - return e1000_acquire_swfw_sync_80003es2lan(hw, mask); + return e1000_acquire_swfw_sync(hw, mask); } /** @@ -353,7 +351,7 @@ static void e1000_release_mac_csr_80003es2lan(struct e1000_hw *hw) mask = E1000_SWFW_CSR_SM; - e1000_release_swfw_sync_80003es2lan(hw, mask); + e1000_release_swfw_sync(hw, mask); } /** @@ -368,14 +366,14 @@ static s32 e1000_acquire_nvm_80003es2lan(struct e1000_hw *hw) DEBUGFUNC("e1000_acquire_nvm_80003es2lan"); - ret_val = e1000_acquire_swfw_sync_80003es2lan(hw, E1000_SWFW_EEP_SM); + ret_val = e1000_acquire_swfw_sync(hw, E1000_SWFW_EEP_SM); if (ret_val) return ret_val; ret_val = e1000_acquire_nvm_generic(hw); if (ret_val) - e1000_release_swfw_sync_80003es2lan(hw, E1000_SWFW_EEP_SM); + e1000_release_swfw_sync(hw, E1000_SWFW_EEP_SM); return ret_val; } @@ -391,78 +389,7 @@ static void e1000_release_nvm_80003es2lan(struct e1000_hw *hw) DEBUGFUNC("e1000_release_nvm_80003es2lan"); e1000_release_nvm_generic(hw); - e1000_release_swfw_sync_80003es2lan(hw, E1000_SWFW_EEP_SM); -} - -/** - * e1000_acquire_swfw_sync_80003es2lan - Acquire SW/FW semaphore - * @hw: pointer to the HW structure - * @mask: specifies which semaphore to acquire - * - * Acquire the SW/FW semaphore to access the PHY or NVM. The mask - * will also specify which port we're acquiring the lock for. - **/ -static s32 e1000_acquire_swfw_sync_80003es2lan(struct e1000_hw *hw, u16 mask) -{ - u32 swfw_sync; - u32 swmask = mask; - u32 fwmask = mask << 16; - s32 i = 0; - s32 timeout = 50; - - DEBUGFUNC("e1000_acquire_swfw_sync_80003es2lan"); - - while (i < timeout) { - if (e1000_get_hw_semaphore_generic(hw)) - return -E1000_ERR_SWFW_SYNC; - - swfw_sync = E1000_READ_REG(hw, E1000_SW_FW_SYNC); - if (!(swfw_sync & (fwmask | swmask))) - break; - - /* Firmware currently using resource (fwmask) - * or other software thread using resource (swmask) - */ - e1000_put_hw_semaphore_generic(hw); - msec_delay_irq(5); - i++; - } - - if (i == timeout) { - DEBUGOUT("Driver can't access resource, SW_FW_SYNC timeout.\n"); - return -E1000_ERR_SWFW_SYNC; - } - - swfw_sync |= swmask; - E1000_WRITE_REG(hw, E1000_SW_FW_SYNC, swfw_sync); - - e1000_put_hw_semaphore_generic(hw); - - return E1000_SUCCESS; -} - -/** - * e1000_release_swfw_sync_80003es2lan - Release SW/FW semaphore - * @hw: pointer to the HW structure - * @mask: specifies which semaphore to acquire - * - * Release the SW/FW semaphore used to access the PHY or NVM. The mask - * will also specify which port we're releasing the lock for. - **/ -static void e1000_release_swfw_sync_80003es2lan(struct e1000_hw *hw, u16 mask) -{ - u32 swfw_sync; - - DEBUGFUNC("e1000_release_swfw_sync_80003es2lan"); - - while (e1000_get_hw_semaphore_generic(hw) != E1000_SUCCESS) - ; /* Empty */ - - swfw_sync = E1000_READ_REG(hw, E1000_SW_FW_SYNC); - swfw_sync &= ~mask; - E1000_WRITE_REG(hw, E1000_SW_FW_SYNC, swfw_sync); - - e1000_put_hw_semaphore_generic(hw); + e1000_release_swfw_sync(hw, E1000_SWFW_EEP_SM); } /** diff --git a/freebsd/sys/dev/e1000/e1000_82571.c b/freebsd/sys/dev/e1000/e1000_82571.c index fdef7284..f84baf92 100644 --- a/freebsd/sys/dev/e1000/e1000_82571.c +++ b/freebsd/sys/dev/e1000/e1000_82571.c @@ -73,11 +73,8 @@ static s32 e1000_check_for_serdes_link_82571(struct e1000_hw *hw); static s32 e1000_setup_fiber_serdes_link_82571(struct e1000_hw *hw); static s32 e1000_valid_led_default_82571(struct e1000_hw *hw, u16 *data); static void e1000_clear_hw_cntrs_82571(struct e1000_hw *hw); -static s32 e1000_get_hw_semaphore_82571(struct e1000_hw *hw); static s32 e1000_fix_nvm_checksum_82571(struct e1000_hw *hw); static s32 e1000_get_phy_id_82571(struct e1000_hw *hw); -static void e1000_put_hw_semaphore_82571(struct e1000_hw *hw); -static void e1000_put_hw_semaphore_82573(struct e1000_hw *hw); static s32 e1000_get_hw_semaphore_82574(struct e1000_hw *hw); static void e1000_put_hw_semaphore_82574(struct e1000_hw *hw); static s32 e1000_set_d0_lplu_state_82574(struct e1000_hw *hw, @@ -128,8 +125,8 @@ static s32 e1000_init_phy_params_82571(struct e1000_hw *hw) phy->ops.get_cable_length = e1000_get_cable_length_igp_2; phy->ops.read_reg = e1000_read_phy_reg_igp; phy->ops.write_reg = e1000_write_phy_reg_igp; - phy->ops.acquire = e1000_get_hw_semaphore_82571; - phy->ops.release = e1000_put_hw_semaphore_82571; + phy->ops.acquire = e1000_get_hw_semaphore; + phy->ops.release = e1000_put_hw_semaphore; break; case e1000_82573: phy->type = e1000_phy_m88; @@ -141,12 +138,11 @@ static s32 e1000_init_phy_params_82571(struct e1000_hw *hw) phy->ops.get_cable_length = e1000_get_cable_length_m88; phy->ops.read_reg = e1000_read_phy_reg_m88; phy->ops.write_reg = e1000_write_phy_reg_m88; - phy->ops.acquire = e1000_get_hw_semaphore_82571; - phy->ops.release = e1000_put_hw_semaphore_82571; + phy->ops.acquire = e1000_get_hw_semaphore; + phy->ops.release = e1000_put_hw_semaphore; break; case e1000_82574: case e1000_82583: - E1000_MUTEX_INIT(&hw->dev_spec._82571.swflag_mutex); phy->type = e1000_phy_bm; phy->ops.get_cfg_done = e1000_get_cfg_done_generic; @@ -509,99 +505,21 @@ static s32 e1000_get_phy_id_82571(struct e1000_hw *hw) } /** - * e1000_get_hw_semaphore_82571 - Acquire hardware semaphore - * @hw: pointer to the HW structure - * - * Acquire the HW semaphore to access the PHY or NVM - **/ -static s32 e1000_get_hw_semaphore_82571(struct e1000_hw *hw) -{ - u32 swsm; - s32 sw_timeout = hw->nvm.word_size + 1; - s32 fw_timeout = hw->nvm.word_size + 1; - s32 i = 0; - - DEBUGFUNC("e1000_get_hw_semaphore_82571"); - - /* If we have timedout 3 times on trying to acquire - * the inter-port SMBI semaphore, there is old code - * operating on the other port, and it is not - * releasing SMBI. Modify the number of times that - * we try for the semaphore to interwork with this - * older code. - */ - if (hw->dev_spec._82571.smb_counter > 2) - sw_timeout = 1; - - /* Get the SW semaphore */ - while (i < sw_timeout) { - swsm = E1000_READ_REG(hw, E1000_SWSM); - if (!(swsm & E1000_SWSM_SMBI)) - break; - - usec_delay(50); - i++; - } - - if (i == sw_timeout) { - DEBUGOUT("Driver can't access device - SMBI bit is set.\n"); - hw->dev_spec._82571.smb_counter++; - } - /* Get the FW semaphore. */ - for (i = 0; i < fw_timeout; i++) { - swsm = E1000_READ_REG(hw, E1000_SWSM); - E1000_WRITE_REG(hw, E1000_SWSM, swsm | E1000_SWSM_SWESMBI); - - /* Semaphore acquired if bit latched */ - if (E1000_READ_REG(hw, E1000_SWSM) & E1000_SWSM_SWESMBI) - break; - - usec_delay(50); - } - - if (i == fw_timeout) { - /* Release semaphores */ - e1000_put_hw_semaphore_82571(hw); - DEBUGOUT("Driver can't access the NVM\n"); - return -E1000_ERR_NVM; - } - - return E1000_SUCCESS; -} - -/** - * e1000_put_hw_semaphore_82571 - Release hardware semaphore - * @hw: pointer to the HW structure - * - * Release hardware semaphore used to access the PHY or NVM - **/ -static void e1000_put_hw_semaphore_82571(struct e1000_hw *hw) -{ - u32 swsm; - - DEBUGFUNC("e1000_put_hw_semaphore_generic"); - - swsm = E1000_READ_REG(hw, E1000_SWSM); - - swsm &= ~(E1000_SWSM_SMBI | E1000_SWSM_SWESMBI); - - E1000_WRITE_REG(hw, E1000_SWSM, swsm); -} - -/** - * e1000_get_hw_semaphore_82573 - Acquire hardware semaphore + * e1000_get_hw_semaphore_82574 - Acquire hardware semaphore * @hw: pointer to the HW structure * * Acquire the HW semaphore during reset. * **/ -static s32 e1000_get_hw_semaphore_82573(struct e1000_hw *hw) +static s32 +e1000_get_hw_semaphore_82574(struct e1000_hw *hw) { u32 extcnf_ctrl; s32 i = 0; - + /* XXX assert that mutex is held */ DEBUGFUNC("e1000_get_hw_semaphore_82573"); + ASSERT_CTX_LOCK_HELD(hw); extcnf_ctrl = E1000_READ_REG(hw, E1000_EXTCNF_CTRL); do { extcnf_ctrl |= E1000_EXTCNF_CTRL_MDIO_SW_OWNERSHIP; @@ -617,7 +535,7 @@ static s32 e1000_get_hw_semaphore_82573(struct e1000_hw *hw) if (i == MDIO_OWNERSHIP_TIMEOUT) { /* Release semaphores */ - e1000_put_hw_semaphore_82573(hw); + e1000_put_hw_semaphore_82574(hw); DEBUGOUT("Driver can't access the PHY\n"); return -E1000_ERR_PHY; } @@ -626,17 +544,18 @@ static s32 e1000_get_hw_semaphore_82573(struct e1000_hw *hw) } /** - * e1000_put_hw_semaphore_82573 - Release hardware semaphore + * e1000_put_hw_semaphore_82574 - Release hardware semaphore * @hw: pointer to the HW structure * * Release hardware semaphore used during reset. * **/ -static void e1000_put_hw_semaphore_82573(struct e1000_hw *hw) +static void +e1000_put_hw_semaphore_82574(struct e1000_hw *hw) { u32 extcnf_ctrl; - DEBUGFUNC("e1000_put_hw_semaphore_82573"); + DEBUGFUNC("e1000_put_hw_semaphore_82574"); extcnf_ctrl = E1000_READ_REG(hw, E1000_EXTCNF_CTRL); extcnf_ctrl &= ~E1000_EXTCNF_CTRL_MDIO_SW_OWNERSHIP; @@ -644,41 +563,6 @@ static void e1000_put_hw_semaphore_82573(struct e1000_hw *hw) } /** - * e1000_get_hw_semaphore_82574 - Acquire hardware semaphore - * @hw: pointer to the HW structure - * - * Acquire the HW semaphore to access the PHY or NVM. - * - **/ -static s32 e1000_get_hw_semaphore_82574(struct e1000_hw *hw) -{ - s32 ret_val; - - DEBUGFUNC("e1000_get_hw_semaphore_82574"); - - E1000_MUTEX_LOCK(&hw->dev_spec._82571.swflag_mutex); - ret_val = e1000_get_hw_semaphore_82573(hw); - if (ret_val) - E1000_MUTEX_UNLOCK(&hw->dev_spec._82571.swflag_mutex); - return ret_val; -} - -/** - * e1000_put_hw_semaphore_82574 - Release hardware semaphore - * @hw: pointer to the HW structure - * - * Release hardware semaphore used to access the PHY or NVM - * - **/ -static void e1000_put_hw_semaphore_82574(struct e1000_hw *hw) -{ - DEBUGFUNC("e1000_put_hw_semaphore_82574"); - - e1000_put_hw_semaphore_82573(hw); - E1000_MUTEX_UNLOCK(&hw->dev_spec._82571.swflag_mutex); -} - -/** * e1000_set_d0_lplu_state_82574 - Set Low Power Linkup D0 state * @hw: pointer to the HW structure * @active: TRUE to enable LPLU, FALSE to disable @@ -749,7 +633,7 @@ static s32 e1000_acquire_nvm_82571(struct e1000_hw *hw) DEBUGFUNC("e1000_acquire_nvm_82571"); - ret_val = e1000_get_hw_semaphore_82571(hw); + ret_val = e1000_get_hw_semaphore(hw); if (ret_val) return ret_val; @@ -762,7 +646,7 @@ static s32 e1000_acquire_nvm_82571(struct e1000_hw *hw) } if (ret_val) - e1000_put_hw_semaphore_82571(hw); + e1000_put_hw_semaphore(hw); return ret_val; } @@ -778,7 +662,7 @@ static void e1000_release_nvm_82571(struct e1000_hw *hw) DEBUGFUNC("e1000_release_nvm_82571"); e1000_release_nvm_generic(hw); - e1000_put_hw_semaphore_82571(hw); + e1000_put_hw_semaphore(hw); } /** @@ -1095,8 +979,6 @@ static s32 e1000_reset_hw_82571(struct e1000_hw *hw) */ switch (hw->mac.type) { case e1000_82573: - ret_val = e1000_get_hw_semaphore_82573(hw); - break; case e1000_82574: case e1000_82583: ret_val = e1000_get_hw_semaphore_82574(hw); @@ -1113,10 +995,6 @@ static s32 e1000_reset_hw_82571(struct e1000_hw *hw) /* Must release MDIO ownership and mutex after MAC reset. */ switch (hw->mac.type) { case e1000_82573: - /* Release mutex only if the hw semaphore is acquired */ - if (!ret_val) - e1000_put_hw_semaphore_82573(hw); - break; case e1000_82574: case e1000_82583: /* Release mutex only if the hw semaphore is acquired */ @@ -1124,6 +1002,7 @@ static s32 e1000_reset_hw_82571(struct e1000_hw *hw) e1000_put_hw_semaphore_82574(hw); break; default: + /* we didn't get the semaphore no need to put it */ break; } diff --git a/freebsd/sys/dev/e1000/e1000_82575.c b/freebsd/sys/dev/e1000/e1000_82575.c index bffa1117..0de11ef5 100644 --- a/freebsd/sys/dev/e1000/e1000_82575.c +++ b/freebsd/sys/dev/e1000/e1000_82575.c @@ -82,11 +82,9 @@ static s32 e1000_valid_led_default_82575(struct e1000_hw *hw, u16 *data); static s32 e1000_write_phy_reg_sgmii_82575(struct e1000_hw *hw, u32 offset, u16 data); static void e1000_clear_hw_cntrs_82575(struct e1000_hw *hw); -static s32 e1000_acquire_swfw_sync_82575(struct e1000_hw *hw, u16 mask); static s32 e1000_get_pcs_speed_and_duplex_82575(struct e1000_hw *hw, u16 *speed, u16 *duplex); static s32 e1000_get_phy_id_82575(struct e1000_hw *hw); -static void e1000_release_swfw_sync_82575(struct e1000_hw *hw, u16 mask); static bool e1000_sgmii_active_82575(struct e1000_hw *hw); static s32 e1000_reset_init_script_82575(struct e1000_hw *hw); static s32 e1000_read_mac_addr_82575(struct e1000_hw *hw); @@ -514,12 +512,8 @@ static s32 e1000_init_mac_params_82575(struct e1000_hw *hw) /* link info */ mac->ops.get_link_up_info = e1000_get_link_up_info_82575; /* acquire SW_FW sync */ - mac->ops.acquire_swfw_sync = e1000_acquire_swfw_sync_82575; - mac->ops.release_swfw_sync = e1000_release_swfw_sync_82575; - if (mac->type >= e1000_i210) { - mac->ops.acquire_swfw_sync = e1000_acquire_swfw_sync_i210; - mac->ops.release_swfw_sync = e1000_release_swfw_sync_i210; - } + mac->ops.acquire_swfw_sync = e1000_acquire_swfw_sync; + mac->ops.release_swfw_sync = e1000_release_swfw_sync; /* set lan id for port to determine which phy lock to use */ hw->mac.ops.set_lan_id(hw); @@ -991,7 +985,7 @@ static s32 e1000_acquire_nvm_82575(struct e1000_hw *hw) DEBUGFUNC("e1000_acquire_nvm_82575"); - ret_val = e1000_acquire_swfw_sync_82575(hw, E1000_SWFW_EEP_SM); + ret_val = e1000_acquire_swfw_sync(hw, E1000_SWFW_EEP_SM); if (ret_val) goto out; @@ -1022,7 +1016,7 @@ static s32 e1000_acquire_nvm_82575(struct e1000_hw *hw) ret_val = e1000_acquire_nvm_generic(hw); if (ret_val) - e1000_release_swfw_sync_82575(hw, E1000_SWFW_EEP_SM); + e1000_release_swfw_sync(hw, E1000_SWFW_EEP_SM); out: return ret_val; @@ -1041,83 +1035,7 @@ static void e1000_release_nvm_82575(struct e1000_hw *hw) e1000_release_nvm_generic(hw); - e1000_release_swfw_sync_82575(hw, E1000_SWFW_EEP_SM); -} - -/** - * e1000_acquire_swfw_sync_82575 - Acquire SW/FW semaphore - * @hw: pointer to the HW structure - * @mask: specifies which semaphore to acquire - * - * Acquire the SW/FW semaphore to access the PHY or NVM. The mask - * will also specify which port we're acquiring the lock for. - **/ -static s32 e1000_acquire_swfw_sync_82575(struct e1000_hw *hw, u16 mask) -{ - u32 swfw_sync; - u32 swmask = mask; - u32 fwmask = mask << 16; - s32 ret_val = E1000_SUCCESS; - s32 i = 0, timeout = 200; - - DEBUGFUNC("e1000_acquire_swfw_sync_82575"); - - while (i < timeout) { - if (e1000_get_hw_semaphore_generic(hw)) { - ret_val = -E1000_ERR_SWFW_SYNC; - goto out; - } - - swfw_sync = E1000_READ_REG(hw, E1000_SW_FW_SYNC); - if (!(swfw_sync & (fwmask | swmask))) - break; - - /* - * Firmware currently using resource (fwmask) - * or other software thread using resource (swmask) - */ - e1000_put_hw_semaphore_generic(hw); - msec_delay_irq(5); - i++; - } - - if (i == timeout) { - DEBUGOUT("Driver can't access resource, SW_FW_SYNC timeout.\n"); - ret_val = -E1000_ERR_SWFW_SYNC; - goto out; - } - - swfw_sync |= swmask; - E1000_WRITE_REG(hw, E1000_SW_FW_SYNC, swfw_sync); - - e1000_put_hw_semaphore_generic(hw); - -out: - return ret_val; -} - -/** - * e1000_release_swfw_sync_82575 - Release SW/FW semaphore - * @hw: pointer to the HW structure - * @mask: specifies which semaphore to acquire - * - * Release the SW/FW semaphore used to access the PHY or NVM. The mask - * will also specify which port we're releasing the lock for. - **/ -static void e1000_release_swfw_sync_82575(struct e1000_hw *hw, u16 mask) -{ - u32 swfw_sync; - - DEBUGFUNC("e1000_release_swfw_sync_82575"); - - while (e1000_get_hw_semaphore_generic(hw) != E1000_SUCCESS) - ; /* Empty */ - - swfw_sync = E1000_READ_REG(hw, E1000_SW_FW_SYNC); - swfw_sync &= ~mask; - E1000_WRITE_REG(hw, E1000_SW_FW_SYNC, swfw_sync); - - e1000_put_hw_semaphore_generic(hw); + e1000_release_swfw_sync(hw, E1000_SWFW_EEP_SM); } /** diff --git a/freebsd/sys/dev/e1000/e1000_hw.h b/freebsd/sys/dev/e1000/e1000_hw.h index 2c17a022..7e4e7f1a 100644 --- a/freebsd/sys/dev/e1000/e1000_hw.h +++ b/freebsd/sys/dev/e1000/e1000_hw.h @@ -944,7 +944,6 @@ struct e1000_dev_spec_82543 { struct e1000_dev_spec_82571 { bool laa_is_present; u32 smb_counter; - E1000_MUTEX swflag_mutex; }; struct e1000_dev_spec_80003es2lan { @@ -968,8 +967,6 @@ enum e1000_ulp_state { struct e1000_dev_spec_ich8lan { bool kmrn_lock_loss_workaround_enabled; struct e1000_shadow_ram shadow_ram[E1000_SHADOW_RAM_WORDS]; - E1000_MUTEX nvm_mutex; - E1000_MUTEX swflag_mutex; bool nvm_k1_enabled; bool disable_k1_off; bool eee_disable; diff --git a/freebsd/sys/dev/e1000/e1000_i210.c b/freebsd/sys/dev/e1000/e1000_i210.c index 5f09e4d2..5ee48810 100644 --- a/freebsd/sys/dev/e1000/e1000_i210.c +++ b/freebsd/sys/dev/e1000/e1000_i210.c @@ -40,7 +40,6 @@ static s32 e1000_acquire_nvm_i210(struct e1000_hw *hw); static void e1000_release_nvm_i210(struct e1000_hw *hw); -static s32 e1000_get_hw_semaphore_i210(struct e1000_hw *hw); static s32 e1000_write_nvm_srwr(struct e1000_hw *hw, u16 offset, u16 words, u16 *data); static s32 e1000_pool_flash_update_done_i210(struct e1000_hw *hw); @@ -61,7 +60,7 @@ static s32 e1000_acquire_nvm_i210(struct e1000_hw *hw) DEBUGFUNC("e1000_acquire_nvm_i210"); - ret_val = e1000_acquire_swfw_sync_i210(hw, E1000_SWFW_EEP_SM); + ret_val = e1000_acquire_swfw_sync(hw, E1000_SWFW_EEP_SM); return ret_val; } @@ -77,152 +76,7 @@ static void e1000_release_nvm_i210(struct e1000_hw *hw) { DEBUGFUNC("e1000_release_nvm_i210"); - e1000_release_swfw_sync_i210(hw, E1000_SWFW_EEP_SM); -} - -/** - * e1000_acquire_swfw_sync_i210 - Acquire SW/FW semaphore - * @hw: pointer to the HW structure - * @mask: specifies which semaphore to acquire - * - * Acquire the SW/FW semaphore to access the PHY or NVM. The mask - * will also specify which port we're acquiring the lock for. - **/ -s32 e1000_acquire_swfw_sync_i210(struct e1000_hw *hw, u16 mask) -{ - u32 swfw_sync; - u32 swmask = mask; - u32 fwmask = mask << 16; - s32 ret_val = E1000_SUCCESS; - s32 i = 0, timeout = 200; /* FIXME: find real value to use here */ - - DEBUGFUNC("e1000_acquire_swfw_sync_i210"); - - while (i < timeout) { - if (e1000_get_hw_semaphore_i210(hw)) { - ret_val = -E1000_ERR_SWFW_SYNC; - goto out; - } - - swfw_sync = E1000_READ_REG(hw, E1000_SW_FW_SYNC); - if (!(swfw_sync & (fwmask | swmask))) - break; - - /* - * Firmware currently using resource (fwmask) - * or other software thread using resource (swmask) - */ - e1000_put_hw_semaphore_generic(hw); - msec_delay_irq(5); - i++; - } - - if (i == timeout) { - DEBUGOUT("Driver can't access resource, SW_FW_SYNC timeout.\n"); - ret_val = -E1000_ERR_SWFW_SYNC; - goto out; - } - - swfw_sync |= swmask; - E1000_WRITE_REG(hw, E1000_SW_FW_SYNC, swfw_sync); - - e1000_put_hw_semaphore_generic(hw); - -out: - return ret_val; -} - -/** - * e1000_release_swfw_sync_i210 - Release SW/FW semaphore - * @hw: pointer to the HW structure - * @mask: specifies which semaphore to acquire - * - * Release the SW/FW semaphore used to access the PHY or NVM. The mask - * will also specify which port we're releasing the lock for. - **/ -void e1000_release_swfw_sync_i210(struct e1000_hw *hw, u16 mask) -{ - u32 swfw_sync; - - DEBUGFUNC("e1000_release_swfw_sync_i210"); - - while (e1000_get_hw_semaphore_i210(hw) != E1000_SUCCESS) - ; /* Empty */ - - swfw_sync = E1000_READ_REG(hw, E1000_SW_FW_SYNC); - swfw_sync &= ~mask; - E1000_WRITE_REG(hw, E1000_SW_FW_SYNC, swfw_sync); - - e1000_put_hw_semaphore_generic(hw); -} - -/** - * e1000_get_hw_semaphore_i210 - Acquire hardware semaphore - * @hw: pointer to the HW structure - * - * Acquire the HW semaphore to access the PHY or NVM - **/ -static s32 e1000_get_hw_semaphore_i210(struct e1000_hw *hw) -{ - u32 swsm; - s32 timeout = hw->nvm.word_size + 1; - s32 i = 0; - - DEBUGFUNC("e1000_get_hw_semaphore_i210"); - - /* Get the SW semaphore */ - while (i < timeout) { - swsm = E1000_READ_REG(hw, E1000_SWSM); - if (!(swsm & E1000_SWSM_SMBI)) - break; - - usec_delay(50); - i++; - } - - if (i == timeout) { - /* In rare circumstances, the SW semaphore may already be held - * unintentionally. Clear the semaphore once before giving up. - */ - if (hw->dev_spec._82575.clear_semaphore_once) { - hw->dev_spec._82575.clear_semaphore_once = FALSE; - e1000_put_hw_semaphore_generic(hw); - for (i = 0; i < timeout; i++) { - swsm = E1000_READ_REG(hw, E1000_SWSM); - if (!(swsm & E1000_SWSM_SMBI)) - break; - - usec_delay(50); - } - } - - /* If we do not have the semaphore here, we have to give up. */ - if (i == timeout) { - DEBUGOUT("Driver can't access device - SMBI bit is set.\n"); - return -E1000_ERR_NVM; - } - } - - /* Get the FW semaphore. */ - for (i = 0; i < timeout; i++) { - swsm = E1000_READ_REG(hw, E1000_SWSM); - E1000_WRITE_REG(hw, E1000_SWSM, swsm | E1000_SWSM_SWESMBI); - - /* Semaphore acquired if bit latched */ - if (E1000_READ_REG(hw, E1000_SWSM) & E1000_SWSM_SWESMBI) - break; - - usec_delay(50); - } - - if (i == timeout) { - /* Release semaphores */ - e1000_put_hw_semaphore_generic(hw); - DEBUGOUT("Driver can't access the NVM\n"); - return -E1000_ERR_NVM; - } - - return E1000_SUCCESS; + e1000_release_swfw_sync(hw, E1000_SWFW_EEP_SM); } /** diff --git a/freebsd/sys/dev/e1000/e1000_i210.h b/freebsd/sys/dev/e1000/e1000_i210.h index c08a0dd7..ed6262b5 100644 --- a/freebsd/sys/dev/e1000/e1000_i210.h +++ b/freebsd/sys/dev/e1000/e1000_i210.h @@ -44,8 +44,6 @@ s32 e1000_write_nvm_srwr_i210(struct e1000_hw *hw, u16 offset, u16 words, u16 *data); s32 e1000_read_nvm_srrd_i210(struct e1000_hw *hw, u16 offset, u16 words, u16 *data); -s32 e1000_acquire_swfw_sync_i210(struct e1000_hw *hw, u16 mask); -void e1000_release_swfw_sync_i210(struct e1000_hw *hw, u16 mask); s32 e1000_read_xmdio_reg(struct e1000_hw *hw, u16 addr, u8 dev_addr, u16 *data); s32 e1000_write_xmdio_reg(struct e1000_hw *hw, u16 addr, u8 dev_addr, diff --git a/freebsd/sys/dev/e1000/e1000_ich8lan.c b/freebsd/sys/dev/e1000/e1000_ich8lan.c index bcd82c47..cbddadd7 100644 --- a/freebsd/sys/dev/e1000/e1000_ich8lan.c +++ b/freebsd/sys/dev/e1000/e1000_ich8lan.c @@ -699,9 +699,6 @@ static s32 e1000_init_nvm_params_ich8lan(struct e1000_hw *hw) dev_spec->shadow_ram[i].value = 0xFFFF; } - E1000_MUTEX_INIT(&dev_spec->nvm_mutex); - E1000_MUTEX_INIT(&dev_spec->swflag_mutex); - /* Function Pointers */ nvm->ops.acquire = e1000_acquire_nvm_ich8lan; nvm->ops.release = e1000_release_nvm_ich8lan; @@ -1854,7 +1851,7 @@ static s32 e1000_acquire_nvm_ich8lan(struct e1000_hw *hw) { DEBUGFUNC("e1000_acquire_nvm_ich8lan"); - E1000_MUTEX_LOCK(&hw->dev_spec.ich8lan.nvm_mutex); + ASSERT_CTX_LOCK_HELD(hw); return E1000_SUCCESS; } @@ -1869,9 +1866,7 @@ static void e1000_release_nvm_ich8lan(struct e1000_hw *hw) { DEBUGFUNC("e1000_release_nvm_ich8lan"); - E1000_MUTEX_UNLOCK(&hw->dev_spec.ich8lan.nvm_mutex); - - return; + ASSERT_CTX_LOCK_HELD(hw); } /** @@ -1888,7 +1883,7 @@ static s32 e1000_acquire_swflag_ich8lan(struct e1000_hw *hw) DEBUGFUNC("e1000_acquire_swflag_ich8lan"); - E1000_MUTEX_LOCK(&hw->dev_spec.ich8lan.swflag_mutex); + ASSERT_CTX_LOCK_HELD(hw); while (timeout) { extcnf_ctrl = E1000_READ_REG(hw, E1000_EXTCNF_CTRL); @@ -1929,9 +1924,6 @@ static s32 e1000_acquire_swflag_ich8lan(struct e1000_hw *hw) } out: - if (ret_val) - E1000_MUTEX_UNLOCK(&hw->dev_spec.ich8lan.swflag_mutex); - return ret_val; } @@ -1956,10 +1948,6 @@ static void e1000_release_swflag_ich8lan(struct e1000_hw *hw) } else { DEBUGOUT("Semaphore unexpectedly released by sw/fw/hw\n"); } - - E1000_MUTEX_UNLOCK(&hw->dev_spec.ich8lan.swflag_mutex); - - return; } /** @@ -5034,8 +5022,6 @@ static s32 e1000_reset_hw_ich8lan(struct e1000_hw *hw) E1000_WRITE_REG(hw, E1000_FEXTNVM3, reg); } - if (!ret_val) - E1000_MUTEX_UNLOCK(&hw->dev_spec.ich8lan.swflag_mutex); if (ctrl & E1000_CTRL_PHY_RST) { ret_val = hw->phy.ops.get_cfg_done(hw); diff --git a/freebsd/sys/dev/e1000/e1000_mac.c b/freebsd/sys/dev/e1000/e1000_mac.c index 581659be..7a46ca5a 100644 --- a/freebsd/sys/dev/e1000/e1000_mac.c +++ b/freebsd/sys/dev/e1000/e1000_mac.c @@ -1710,76 +1710,6 @@ s32 e1000_get_speed_and_duplex_fiber_serdes_generic(struct e1000_hw E1000_UNUSED } /** - * e1000_get_hw_semaphore_generic - Acquire hardware semaphore - * @hw: pointer to the HW structure - * - * Acquire the HW semaphore to access the PHY or NVM - **/ -s32 e1000_get_hw_semaphore_generic(struct e1000_hw *hw) -{ - u32 swsm; - s32 timeout = hw->nvm.word_size + 1; - s32 i = 0; - - DEBUGFUNC("e1000_get_hw_semaphore_generic"); - - /* Get the SW semaphore */ - while (i < timeout) { - swsm = E1000_READ_REG(hw, E1000_SWSM); - if (!(swsm & E1000_SWSM_SMBI)) - break; - - usec_delay(50); - i++; - } - - if (i == timeout) { - DEBUGOUT("Driver can't access device - SMBI bit is set.\n"); - return -E1000_ERR_NVM; - } - - /* Get the FW semaphore. */ - for (i = 0; i < timeout; i++) { - swsm = E1000_READ_REG(hw, E1000_SWSM); - E1000_WRITE_REG(hw, E1000_SWSM, swsm | E1000_SWSM_SWESMBI); - - /* Semaphore acquired if bit latched */ - if (E1000_READ_REG(hw, E1000_SWSM) & E1000_SWSM_SWESMBI) - break; - - usec_delay(50); - } - - if (i == timeout) { - /* Release semaphores */ - e1000_put_hw_semaphore_generic(hw); - DEBUGOUT("Driver can't access the NVM\n"); - return -E1000_ERR_NVM; - } - - return E1000_SUCCESS; -} - -/** - * e1000_put_hw_semaphore_generic - Release hardware semaphore - * @hw: pointer to the HW structure - * - * Release hardware semaphore used to access the PHY or NVM - **/ -void e1000_put_hw_semaphore_generic(struct e1000_hw *hw) -{ - u32 swsm; - - DEBUGFUNC("e1000_put_hw_semaphore_generic"); - - swsm = E1000_READ_REG(hw, E1000_SWSM); - - swsm &= ~(E1000_SWSM_SMBI | E1000_SWSM_SWESMBI); - - E1000_WRITE_REG(hw, E1000_SWSM, swsm); -} - -/** * e1000_get_auto_rd_done_generic - Check for auto read completion * @hw: pointer to the HW structure * @@ -2254,3 +2184,181 @@ s32 e1000_write_8bit_ctrl_reg_generic(struct e1000_hw *hw, u32 reg, return E1000_SUCCESS; } + +/** + * e1000_get_hw_semaphore - Acquire hardware semaphore + * @hw: pointer to the HW structure + * + * Acquire the HW semaphore to access the PHY or NVM + **/ +s32 e1000_get_hw_semaphore(struct e1000_hw *hw) +{ + u32 swsm; + s32 fw_timeout = hw->nvm.word_size + 1; + s32 sw_timeout = hw->nvm.word_size + 1; + s32 i = 0; + + DEBUGFUNC("e1000_get_hw_semaphore"); + + /* _82571 */ + /* If we have timedout 3 times on trying to acquire + * the inter-port SMBI semaphore, there is old code + * operating on the other port, and it is not + * releasing SMBI. Modify the number of times that + * we try for the semaphore to interwork with this + * older code. + */ + if (hw->dev_spec._82571.smb_counter > 2) + sw_timeout = 1; + + + /* Get the SW semaphore */ + while (i < sw_timeout) { + swsm = E1000_READ_REG(hw, E1000_SWSM); + if (!(swsm & E1000_SWSM_SMBI)) + break; + + usec_delay(50); + i++; + } + + if (i == sw_timeout) { + DEBUGOUT("Driver can't access device - SMBI bit is set.\n"); + hw->dev_spec._82571.smb_counter++; + } + + /* In rare circumstances, the SW semaphore may already be held + * unintentionally. Clear the semaphore once before giving up. + */ + if (hw->dev_spec._82575.clear_semaphore_once) { + hw->dev_spec._82575.clear_semaphore_once = FALSE; + e1000_put_hw_semaphore(hw); + for (i = 0; i < fw_timeout; i++) { + swsm = E1000_READ_REG(hw, E1000_SWSM); + if (!(swsm & E1000_SWSM_SMBI)) + break; + + usec_delay(50); + } + } + + /* Get the FW semaphore. */ + for (i = 0; i < fw_timeout; i++) { + swsm = E1000_READ_REG(hw, E1000_SWSM); + E1000_WRITE_REG(hw, E1000_SWSM, swsm | E1000_SWSM_SWESMBI); + + /* Semaphore acquired if bit latched */ + if (E1000_READ_REG(hw, E1000_SWSM) & E1000_SWSM_SWESMBI) + break; + + usec_delay(50); + } + + if (i == fw_timeout) { + /* Release semaphores */ + e1000_put_hw_semaphore(hw); + DEBUGOUT("Driver can't access the NVM\n"); + return -E1000_ERR_NVM; + } + + return E1000_SUCCESS; +} + +/** + * e1000_put_hw_semaphore - Release hardware semaphore + * @hw: pointer to the HW structure + * + * Release hardware semaphore used to access the PHY or NVM + **/ +void e1000_put_hw_semaphore(struct e1000_hw *hw) +{ + u32 swsm; + + DEBUGFUNC("e1000_put_hw_semaphore"); + + swsm = E1000_READ_REG(hw, E1000_SWSM); + + swsm &= ~(E1000_SWSM_SMBI | E1000_SWSM_SWESMBI); + + E1000_WRITE_REG(hw, E1000_SWSM, swsm); +} + + +/** + * e1000_acquire_swfw_sync - Acquire SW/FW semaphore + * @hw: pointer to the HW structure + * @mask: specifies which semaphore to acquire + * + * Acquire the SW/FW semaphore to access the PHY or NVM. The mask + * will also specify which port we're acquiring the lock for. + **/ +s32 +e1000_acquire_swfw_sync(struct e1000_hw *hw, u16 mask) +{ + u32 swfw_sync; + u32 swmask = mask; + u32 fwmask = mask << 16; + s32 ret_val = E1000_SUCCESS; + s32 i = 0, timeout = 200; + + DEBUGFUNC("e1000_acquire_swfw_sync"); + ASSERT_NO_LOCKS(); + while (i < timeout) { + if (e1000_get_hw_semaphore(hw)) { + ret_val = -E1000_ERR_SWFW_SYNC; + goto out; + } + + swfw_sync = E1000_READ_REG(hw, E1000_SW_FW_SYNC); + if (!(swfw_sync & (fwmask | swmask))) + break; + + /* + * Firmware currently using resource (fwmask) + * or other software thread using resource (swmask) + */ + e1000_put_hw_semaphore(hw); + msec_delay_irq(5); + i++; + } + + if (i == timeout) { + DEBUGOUT("Driver can't access resource, SW_FW_SYNC timeout.\n"); + ret_val = -E1000_ERR_SWFW_SYNC; + goto out; + } + + swfw_sync |= swmask; + E1000_WRITE_REG(hw, E1000_SW_FW_SYNC, swfw_sync); + + e1000_put_hw_semaphore(hw); + +out: + return ret_val; +} + +/** + * e1000_release_swfw_sync - Release SW/FW semaphore + * @hw: pointer to the HW structure + * @mask: specifies which semaphore to acquire + * + * Release the SW/FW semaphore used to access the PHY or NVM. The mask + * will also specify which port we're releasing the lock for. + **/ +void +e1000_release_swfw_sync(struct e1000_hw *hw, u16 mask) +{ + u32 swfw_sync; + + DEBUGFUNC("e1000_release_swfw_sync"); + + while (e1000_get_hw_semaphore(hw) != E1000_SUCCESS) + ; /* Empty */ + + swfw_sync = E1000_READ_REG(hw, E1000_SW_FW_SYNC); + swfw_sync &= ~mask; + E1000_WRITE_REG(hw, E1000_SW_FW_SYNC, swfw_sync); + + e1000_put_hw_semaphore(hw); +} + diff --git a/freebsd/sys/dev/e1000/e1000_mac.h b/freebsd/sys/dev/e1000/e1000_mac.h index cb8da246..66f94595 100644 --- a/freebsd/sys/dev/e1000/e1000_mac.h +++ b/freebsd/sys/dev/e1000/e1000_mac.h @@ -61,7 +61,6 @@ s32 e1000_get_bus_info_pci_generic(struct e1000_hw *hw); s32 e1000_get_bus_info_pcie_generic(struct e1000_hw *hw); void e1000_set_lan_id_single_port(struct e1000_hw *hw); void e1000_set_lan_id_multi_port_pci(struct e1000_hw *hw); -s32 e1000_get_hw_semaphore_generic(struct e1000_hw *hw); s32 e1000_get_speed_and_duplex_copper_generic(struct e1000_hw *hw, u16 *speed, u16 *duplex); s32 e1000_get_speed_and_duplex_fiber_serdes_generic(struct e1000_hw *hw, @@ -86,11 +85,15 @@ void e1000_clear_hw_cntrs_base_generic(struct e1000_hw *hw); void e1000_clear_vfta_generic(struct e1000_hw *hw); void e1000_init_rx_addrs_generic(struct e1000_hw *hw, u16 rar_count); void e1000_pcix_mmrbc_workaround_generic(struct e1000_hw *hw); -void e1000_put_hw_semaphore_generic(struct e1000_hw *hw); s32 e1000_check_alt_mac_addr_generic(struct e1000_hw *hw); void e1000_reset_adaptive_generic(struct e1000_hw *hw); void e1000_set_pcie_no_snoop_generic(struct e1000_hw *hw, u32 no_snoop); void e1000_update_adaptive_generic(struct e1000_hw *hw); void e1000_write_vfta_generic(struct e1000_hw *hw, u32 offset, u32 value); +s32 e1000_get_hw_semaphore(struct e1000_hw *hw); +void e1000_put_hw_semaphore(struct e1000_hw *hw); +s32 e1000_acquire_swfw_sync(struct e1000_hw *hw, u16 mask); +void e1000_release_swfw_sync(struct e1000_hw *hw, u16 mask); + #endif diff --git a/freebsd/sys/dev/e1000/e1000_osdep.h b/freebsd/sys/dev/e1000/e1000_osdep.h index d9f956f3..55348785 100644 --- a/freebsd/sys/dev/e1000/e1000_osdep.h +++ b/freebsd/sys/dev/e1000/e1000_osdep.h @@ -40,6 +40,7 @@ #include <sys/types.h> #include <sys/param.h> #include <sys/systm.h> +#include <sys/proc.h> #include <sys/lock.h> #include <sys/mutex.h> #include <sys/mbuf.h> @@ -48,6 +49,14 @@ #include <sys/malloc.h> #include <sys/kernel.h> #include <sys/bus.h> + +#include <net/ethernet.h> +#include <net/if.h> +#include <net/if_var.h> +#include <net/iflib.h> + + + #include <machine/bus.h> #include <sys/rman.h> #include <machine/resource.h> @@ -59,11 +68,40 @@ #define ASSERT(x) if(!(x)) panic("EM: x") +#define us_scale(x) max(1, (x/(1000000/hz))) +static inline int +ms_scale(int x) { + if (hz == 1000) { + return (x); + } else if (hz > 1000) { + return (x*(hz/1000)); + } else { + return (max(1, x/(1000/hz))); + } +} + +static inline void +safe_pause_us(int x) { + if (cold) { + DELAY(x); + } else { + pause("e1000_delay", max(1, x/(1000000/hz))); + } +} + +static inline void +safe_pause_ms(int x) { + if (cold) { + DELAY(x*1000); + } else { + pause("e1000_delay", ms_scale(x)); + } +} -#define usec_delay(x) DELAY(x) +#define usec_delay(x) safe_pause_us(x) #define usec_delay_irq(x) usec_delay(x) -#define msec_delay(x) DELAY(1000*(x)) -#define msec_delay_irq(x) DELAY(1000*(x)) +#define msec_delay(x) safe_pause_ms(x) +#define msec_delay_irq(x) msec_delay(x) /* Enable/disable debugging statements in shared code */ #define DBG 0 @@ -82,16 +120,6 @@ #define CMD_MEM_WRT_INVALIDATE 0x0010 /* BIT_4 */ #define PCI_COMMAND_REGISTER PCIR_COMMAND -/* Mutex used in the shared code */ -#define E1000_MUTEX struct mtx -#define E1000_MUTEX_INIT(mutex) mtx_init((mutex), #mutex, \ - MTX_NETWORK_LOCK, \ - MTX_DEF | MTX_DUPOK) -#define E1000_MUTEX_DESTROY(mutex) mtx_destroy(mutex) -#define E1000_MUTEX_LOCK(mutex) mtx_lock(mutex) -#define E1000_MUTEX_TRYLOCK(mutex) mtx_trylock(mutex) -#define E1000_MUTEX_UNLOCK(mutex) mtx_unlock(mutex) - typedef uint64_t u64; typedef uint32_t u32; typedef uint16_t u16; @@ -117,6 +145,12 @@ typedef int8_t s8; #endif #endif /*__FreeBSD_version < 800000 */ +#ifdef INVARIANTS +#define ASSERT_CTX_LOCK_HELD(hw) (sx_assert(iflib_ctx_lock_get(((struct e1000_osdep *)hw->back)->ctx), SX_XLOCKED)) +#else +#define ASSERT_CTX_LOCK_HELD(hw) +#endif + #if defined(__i386__) || defined(__amd64__) static __inline void prefetch(void *x) @@ -136,6 +170,7 @@ struct e1000_osdep bus_space_tag_t flash_bus_space_tag; bus_space_handle_t flash_bus_space_handle; device_t dev; + if_ctx_t ctx; }; #define E1000_REGISTER(hw, reg) (((hw)->mac.type >= e1000_82543) \ @@ -217,5 +252,22 @@ struct e1000_osdep bus_space_write_2(((struct e1000_osdep *)(hw)->back)->flash_bus_space_tag, \ ((struct e1000_osdep *)(hw)->back)->flash_bus_space_handle, reg, value) + +#if defined(INVARIANTS) +#include <sys/proc.h> + +#define ASSERT_NO_LOCKS() \ + do { \ + int unknown_locks = curthread->td_locks - mtx_owned(&Giant); \ + if (unknown_locks > 0) { \ + WITNESS_WARN(WARN_GIANTOK|WARN_SLEEPOK|WARN_PANIC, NULL, "unexpected non-sleepable lock"); \ + } \ + MPASS(curthread->td_rw_rlocks == 0); \ + MPASS(curthread->td_lk_slocks == 0); \ + } while (0) +#else +#define ASSERT_NO_LOCKS() +#endif + #endif /* _FREEBSD_OS_H_ */ diff --git a/freebsd/sys/dev/e1000/em_txrx.c b/freebsd/sys/dev/e1000/em_txrx.c index 92b4a5f4..8157c9ce 100644 --- a/freebsd/sys/dev/e1000/em_txrx.c +++ b/freebsd/sys/dev/e1000/em_txrx.c @@ -70,25 +70,25 @@ static int em_determine_rsstype(u32 pkt_info); extern int em_intr(void *arg); struct if_txrx em_txrx = { - em_isc_txd_encap, - em_isc_txd_flush, - em_isc_txd_credits_update, - em_isc_rxd_available, - em_isc_rxd_pkt_get, - em_isc_rxd_refill, - em_isc_rxd_flush, - em_intr + .ift_txd_encap = em_isc_txd_encap, + .ift_txd_flush = em_isc_txd_flush, + .ift_txd_credits_update = em_isc_txd_credits_update, + .ift_rxd_available = em_isc_rxd_available, + .ift_rxd_pkt_get = em_isc_rxd_pkt_get, + .ift_rxd_refill = em_isc_rxd_refill, + .ift_rxd_flush = em_isc_rxd_flush, + .ift_legacy_intr = em_intr }; struct if_txrx lem_txrx = { - em_isc_txd_encap, - em_isc_txd_flush, - em_isc_txd_credits_update, - lem_isc_rxd_available, - lem_isc_rxd_pkt_get, - lem_isc_rxd_refill, - em_isc_rxd_flush, - em_intr + .ift_txd_encap = em_isc_txd_encap, + .ift_txd_flush = em_isc_txd_flush, + .ift_txd_credits_update = em_isc_txd_credits_update, + .ift_rxd_available = lem_isc_rxd_available, + .ift_rxd_pkt_get = lem_isc_rxd_pkt_get, + .ift_rxd_refill = lem_isc_rxd_refill, + .ift_rxd_flush = em_isc_rxd_flush, + .ift_legacy_intr = em_intr }; extern if_shared_ctx_t em_sctx; @@ -403,7 +403,7 @@ em_isc_txd_encap(void *arg, if_pkt_info_t pi) * needs End Of Packet (EOP) * and Report Status (RS) */ - if (txd_flags) { + if (txd_flags && nsegs) { txr->tx_rsq[txr->tx_rs_pidx] = pidx_last; DPRINTF(iflib_get_dev(sc->ctx), "setting to RS on %d rs_pidx %d first: %d\n", pidx_last, txr->tx_rs_pidx, first); txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1); diff --git a/freebsd/sys/dev/e1000/if_em.c b/freebsd/sys/dev/e1000/if_em.c index 5150eaea..72711a6e 100644 --- a/freebsd/sys/dev/e1000/if_em.c +++ b/freebsd/sys/dev/e1000/if_em.c @@ -738,7 +738,7 @@ em_if_attach_pre(if_ctx_t ctx) return (ENXIO); } - adapter->ctx = ctx; + adapter->ctx = adapter->osdep.ctx = ctx; adapter->dev = adapter->osdep.dev = dev; scctx = adapter->shared = iflib_get_softc_ctx(ctx); adapter->media = iflib_get_media(ctx); @@ -1686,13 +1686,6 @@ em_if_timer(if_ctx_t ctx, uint16_t qid) return; iflib_admin_intr_deferred(ctx); - /* Reset LAA into RAR[0] on 82571 */ - if ((adapter->hw.mac.type == e1000_82571) && - e1000_get_laa_state_82571(&adapter->hw)) - e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); - - if (adapter->hw.mac.type < em_mac_min) - lem_smartspeed(adapter); /* Mask to use in the irq trigger */ if (adapter->intr_type == IFLIB_INTR_MSIX) { @@ -1803,6 +1796,14 @@ em_if_update_admin_status(if_ctx_t ctx) } em_update_stats_counters(adapter); + /* Reset LAA into RAR[0] on 82571 */ + if ((adapter->hw.mac.type == e1000_82571) && + e1000_get_laa_state_82571(&adapter->hw)) + e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); + + if (adapter->hw.mac.type < em_mac_min) + lem_smartspeed(adapter); + E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_LINK | E1000_IMS_LSC); } diff --git a/freebsd/sys/dev/e1000/igb_txrx.c b/freebsd/sys/dev/e1000/igb_txrx.c index 05b2fff2..3a56a496 100644 --- a/freebsd/sys/dev/e1000/igb_txrx.c +++ b/freebsd/sys/dev/e1000/igb_txrx.c @@ -64,14 +64,14 @@ extern void igb_if_enable_intr(if_ctx_t ctx); extern int em_intr(void *arg); struct if_txrx igb_txrx = { - igb_isc_txd_encap, - igb_isc_txd_flush, - igb_isc_txd_credits_update, - igb_isc_rxd_available, - igb_isc_rxd_pkt_get, - igb_isc_rxd_refill, - igb_isc_rxd_flush, - em_intr + .ift_txd_encap = igb_isc_txd_encap, + .ift_txd_flush = igb_isc_txd_flush, + .ift_txd_credits_update = igb_isc_txd_credits_update, + .ift_rxd_available = igb_isc_rxd_available, + .ift_rxd_pkt_get = igb_isc_rxd_pkt_get, + .ift_rxd_refill = igb_isc_rxd_refill, + .ift_rxd_flush = igb_isc_rxd_flush, + .ift_legacy_intr = em_intr }; extern if_shared_ctx_t em_sctx; diff --git a/freebsd/sys/dev/evdev/uinput.c b/freebsd/sys/dev/evdev/uinput.c index f1f812cc..3b332d1f 100644 --- a/freebsd/sys/dev/evdev/uinput.c +++ b/freebsd/sys/dev/evdev/uinput.c @@ -606,6 +606,15 @@ uinput_ioctl_sub(struct uinput_cdev_state *state, u_long cmd, caddr_t data) evdev_set_phys(state->ucs_evdev, buf); return (0); + case UI_SET_BSDUNIQ: + if (state->ucs_state == UINPUT_RUNNING) + return (EINVAL); + ret = copyinstr(*(void **)data, buf, sizeof(buf), NULL); + if (ret != 0) + return (ret); + evdev_set_serial(state->ucs_evdev, buf); + return (0); + case UI_SET_SWBIT: if (state->ucs_state == UINPUT_RUNNING || intdata > SW_MAX || intdata < 0) diff --git a/freebsd/sys/dev/evdev/uinput.h b/freebsd/sys/dev/evdev/uinput.h index dd4b0a82..d15b8945 100644 --- a/freebsd/sys/dev/evdev/uinput.h +++ b/freebsd/sys/dev/evdev/uinput.h @@ -90,6 +90,13 @@ struct uinput_abs_setup { #define UI_BEGIN_FF_ERASE _IOWR(UINPUT_IOCTL_BASE, 202, struct uinput_ff_erase) #define UI_END_FF_ERASE _IOW(UINPUT_IOCTL_BASE, 203, struct uinput_ff_erase) +/* + * FreeBSD specific. Set unique identifier of input device. + * Name and magic are chosen to reduce chances of clashing + * with possible future Linux extensions. + */ +#define UI_SET_BSDUNIQ _IO(UINPUT_IOCTL_BASE, 109) + #define EV_UINPUT 0x0101 #define UI_FF_UPLOAD 1 #define UI_FF_ERASE 2 diff --git a/freebsd/sys/dev/fdt/fdt_common.c b/freebsd/sys/dev/fdt/fdt_common.c index de4a44b9..ff32dc0a 100644 --- a/freebsd/sys/dev/fdt/fdt_common.c +++ b/freebsd/sys/dev/fdt/fdt_common.c @@ -61,7 +61,6 @@ __FBSDID("$FreeBSD$"); #endif #define FDT_COMPAT_LEN 255 -#define FDT_TYPE_LEN 64 #define FDT_REG_CELLS 4 #define FDT_RANGES_SIZE 48 @@ -74,8 +73,6 @@ vm_offset_t fdt_immr_size; struct fdt_ic_list fdt_ic_list_head = SLIST_HEAD_INITIALIZER(fdt_ic_list_head); -static int fdt_is_compatible(phandle_t, const char *); - static int fdt_get_range_by_busaddr(phandle_t node, u_long addr, u_long *base, u_long *size) @@ -218,7 +215,7 @@ fdt_immr_addr(vm_offset_t immr_va) * Try to access the SOC node directly i.e. through /aliases/. */ if ((node = OF_finddevice("soc")) != -1) - if (fdt_is_compatible(node, "simple-bus")) + if (ofw_bus_node_is_compatible(node, "simple-bus")) goto moveon; /* * Find the node the long way. @@ -239,45 +236,6 @@ moveon: return (r); } -/* - * This routine is an early-usage version of the ofw_bus_is_compatible() when - * the ofw_bus I/F is not available (like early console routines and similar). - * Note the buffer has to be on the stack since malloc() is usually not - * available in such cases either. - */ -static int -fdt_is_compatible(phandle_t node, const char *compatstr) -{ - char buf[FDT_COMPAT_LEN]; - char *compat; - int len, onelen, l, rv; - - if ((len = OF_getproplen(node, "compatible")) <= 0) - return (0); - - compat = (char *)&buf; - bzero(compat, FDT_COMPAT_LEN); - - if (OF_getprop(node, "compatible", compat, FDT_COMPAT_LEN) < 0) - return (0); - - onelen = strlen(compatstr); - rv = 0; - while (len > 0) { - if (strncasecmp(compat, compatstr, onelen) == 0) { - /* Found it. */ - rv = 1; - break; - } - /* Slide to the next sub-string. */ - l = strlen(compat) + 1; - compat += l; - len -= l; - } - - return (rv); -} - int fdt_is_compatible_strict(phandle_t node, const char *compatible) { @@ -306,7 +264,7 @@ fdt_find_compatible(phandle_t start, const char *compat, int strict) * matching 'compatible' property. */ for (child = OF_child(start); child != 0; child = OF_peer(child)) - if (fdt_is_compatible(child, compat)) { + if (ofw_bus_node_is_compatible(child, compat)) { if (strict) if (!fdt_is_compatible_strict(child, compat)) continue; @@ -325,7 +283,7 @@ fdt_depth_search_compatible(phandle_t start, const char *compat, int strict) * matching 'compatible' property. */ for (node = OF_child(start); node != 0; node = OF_peer(node)) { - if (fdt_is_compatible(node, compat) && + if (ofw_bus_node_is_compatible(node, compat) && (strict == 0 || fdt_is_compatible_strict(node, compat))) { return (node); } @@ -337,46 +295,6 @@ fdt_depth_search_compatible(phandle_t start, const char *compat, int strict) } int -fdt_is_enabled(phandle_t node) -{ - char *stat; - int ena, len; - - len = OF_getprop_alloc(node, "status", sizeof(char), - (void **)&stat); - - if (len <= 0) - /* It is OK if no 'status' property. */ - return (1); - - /* Anything other than 'okay' means disabled. */ - ena = 0; - if (strncmp((char *)stat, "okay", len) == 0) - ena = 1; - - OF_prop_free(stat); - return (ena); -} - -int -fdt_is_type(phandle_t node, const char *typestr) -{ - char type[FDT_TYPE_LEN]; - - if (OF_getproplen(node, "device_type") <= 0) - return (0); - - if (OF_getprop(node, "device_type", type, FDT_TYPE_LEN) < 0) - return (0); - - if (strncasecmp(type, typestr, FDT_TYPE_LEN) == 0) - /* This fits. */ - return (1); - - return (0); -} - -int fdt_parent_addr_cells(phandle_t node) { pcell_t addr_cells; @@ -389,19 +307,6 @@ fdt_parent_addr_cells(phandle_t node) return ((int)fdt32_to_cpu(addr_cells)); } -int -fdt_pm_is_enabled(phandle_t node) -{ - int ret; - - ret = 1; - -#if defined(SOC_MV_KIRKWOOD) || defined(SOC_MV_DISCOVERY) - ret = fdt_pm(node); -#endif - return (ret); -} - u_long fdt_data_get(void *data, int cells) { @@ -477,59 +382,6 @@ fdt_regsize(phandle_t node, u_long *base, u_long *size) } int -fdt_reg_to_rl(phandle_t node, struct resource_list *rl) -{ - u_long end, count, start; - pcell_t *reg, *regptr; - pcell_t addr_cells, size_cells; - int tuple_size, tuples; - int i, rv; - long busaddr, bussize; - - if (fdt_addrsize_cells(OF_parent(node), &addr_cells, &size_cells) != 0) - return (ENXIO); - if (fdt_get_range(OF_parent(node), 0, &busaddr, &bussize)) { - busaddr = 0; - bussize = 0; - } - - tuple_size = sizeof(pcell_t) * (addr_cells + size_cells); - tuples = OF_getprop_alloc(node, "reg", tuple_size, (void **)®); - debugf("addr_cells = %d, size_cells = %d\n", addr_cells, size_cells); - debugf("tuples = %d, tuple size = %d\n", tuples, tuple_size); - if (tuples <= 0) - /* No 'reg' property in this node. */ - return (0); - - regptr = reg; - for (i = 0; i < tuples; i++) { - - rv = fdt_data_to_res(reg, addr_cells, size_cells, &start, - &count); - if (rv != 0) { - resource_list_free(rl); - goto out; - } - reg += addr_cells + size_cells; - - /* Calculate address range relative to base. */ - start += busaddr; - end = start + count - 1; - - debugf("reg addr start = %lx, end = %lx, count = %lx\n", start, - end, count); - - resource_list_add(rl, SYS_RES_MEMORY, i, start, end, - count); - } - rv = 0; - -out: - OF_prop_free(regptr); - return (rv); -} - -int fdt_get_phyaddr(phandle_t node, device_t dev, int *phy_addr, void **phy_sc) { phandle_t phy_node; @@ -650,6 +502,47 @@ out: } int +fdt_get_reserved_mem(struct mem_region *reserved, int *mreserved) +{ + pcell_t reg[FDT_REG_CELLS]; + phandle_t child, root; + int addr_cells, size_cells; + int i, rv; + + root = OF_finddevice("/reserved-memory"); + if (root == -1) { + return (ENXIO); + } + + if ((rv = fdt_addrsize_cells(root, &addr_cells, &size_cells)) != 0) + return (rv); + + if (addr_cells + size_cells > FDT_REG_CELLS) + panic("Too many address and size cells %d %d", addr_cells, + size_cells); + + i = 0; + for (child = OF_child(root); child != 0; child = OF_peer(child)) { + if (!OF_hasprop(child, "no-map")) + continue; + + rv = OF_getprop(child, "reg", reg, sizeof(reg)); + if (rv <= 0) + /* XXX: Does a no-map of a dynamic range make sense? */ + continue; + + fdt_data_to_res(reg, addr_cells, size_cells, + (u_long *)&reserved[i].mr_start, + (u_long *)&reserved[i].mr_size); + i++; + } + + *mreserved = i; + + return (0); +} + +int fdt_get_mem_regions(struct mem_region *mr, int *mrcnt, uint64_t *memsize) { pcell_t reg[FDT_REG_CELLS * FDT_MEM_REGIONS]; @@ -715,17 +608,6 @@ out: } int -fdt_get_unit(device_t dev) -{ - const char * name; - - name = ofw_bus_get_name(dev); - name = strchr(name, '@') + 1; - - return (strtol(name,NULL,0)); -} - -int fdt_get_chosen_bootargs(char *bootargs, size_t max_size) { phandle_t chosen; diff --git a/freebsd/sys/dev/fdt/fdt_common.h b/freebsd/sys/dev/fdt/fdt_common.h index 91522df5..3bf4df41 100644 --- a/freebsd/sys/dev/fdt/fdt_common.h +++ b/freebsd/sys/dev/fdt/fdt_common.h @@ -39,7 +39,7 @@ #include <contrib/libfdt/libfdt_env.h> #include <dev/ofw/ofw_bus.h> -#define FDT_MEM_REGIONS 8 +#define FDT_MEM_REGIONS 16 #define DI_MAX_INTR_NUM 32 @@ -85,19 +85,14 @@ int fdt_data_to_res(pcell_t *, int, int, u_long *, u_long *); phandle_t fdt_find_compatible(phandle_t, const char *, int); phandle_t fdt_depth_search_compatible(phandle_t, const char *, int); int fdt_get_mem_regions(struct mem_region *, int *, uint64_t *); +int fdt_get_reserved_mem(struct mem_region *, int *); int fdt_get_reserved_regions(struct mem_region *, int *); int fdt_get_phyaddr(phandle_t, device_t, int *, void **); int fdt_get_range(phandle_t, int, u_long *, u_long *); int fdt_immr_addr(vm_offset_t); int fdt_regsize(phandle_t, u_long *, u_long *); int fdt_is_compatible_strict(phandle_t, const char *); -int fdt_is_enabled(phandle_t); -int fdt_pm_is_enabled(phandle_t); -int fdt_is_type(phandle_t, const char *); int fdt_parent_addr_cells(phandle_t); -int fdt_reg_to_rl(phandle_t, struct resource_list *); -int fdt_pm(phandle_t); -int fdt_get_unit(device_t); int fdt_get_chosen_bootargs(char *bootargs, size_t max_size); #endif /* _FDT_COMMON_H_ */ diff --git a/freebsd/sys/dev/ffec/if_ffec.c b/freebsd/sys/dev/ffec/if_ffec.c index 1d842286..03dca1a9 100644 --- a/freebsd/sys/dev/ffec/if_ffec.c +++ b/freebsd/sys/dev/ffec/if_ffec.c @@ -1093,7 +1093,7 @@ ffec_setup_rxfilter(struct ffec_softc *sc) else { ghash = 0; if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &sc->ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &sc->ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; /* 6 bits from MSB in LE CRC32 are used for hash. */ diff --git a/freebsd/sys/dev/gpio/ofw_gpiobus.c b/freebsd/sys/dev/gpio/ofw_gpiobus.c index 1a87121c..ac0ea9bf 100644 --- a/freebsd/sys/dev/gpio/ofw_gpiobus.c +++ b/freebsd/sys/dev/gpio/ofw_gpiobus.c @@ -370,7 +370,7 @@ ofw_gpiobus_parse_gpios_impl(device_t consumer, phandle_t cnode, char *pname, pcell_t *gpios; phandle_t gpio; - ncells = OF_getencprop_alloc(cnode, pname, sizeof(*gpios), + ncells = OF_getencprop_alloc_multi(cnode, pname, sizeof(*gpios), (void **)&gpios); if (ncells == -1) { device_printf(consumer, diff --git a/freebsd/sys/dev/mmc/mmcreg.h b/freebsd/sys/dev/mmc/mmcreg.h index 5f58ce3f..4b1f8a0e 100644 --- a/freebsd/sys/dev/mmc/mmcreg.h +++ b/freebsd/sys/dev/mmc/mmcreg.h @@ -162,34 +162,35 @@ struct mmc_command { #define R1_STATE_PRG 7 #define R1_STATE_DIS 8 -/* R4 response (SDIO) */ -#define R4_IO_NUM_FUNCTIONS(ocr) (((ocr) >> 28) & 0x3) -#define R4_IO_MEM_PRESENT (0x1<<27) -#define R4_IO_OCR_MASK 0x00fffff0 +/* R4 responses (SDIO) */ +#define R4_IO_NUM_FUNCTIONS(ocr) (((ocr) >> 28) & 0x3) +#define R4_IO_MEM_PRESENT (0x1 << 27) +#define R4_IO_OCR_MASK 0x00fffff0 /* * R5 responses * * Types (per SD 2.0 standard) - *e : error bit - *s : status bit - *r : detected and set for the actual command response - *x : Detected and set during command execution. The host can get - * the status by issuing a command with R1 response. + * e : error bit + * s : status bit + * r : detected and set for the actual command response + * x : Detected and set during command execution. The host can get + * the status by issuing a command with R1 response. * * Clear Condition (per SD 2.0 standard) - *a : according to the card current state. - *b : always related to the previous command. reception of a valid - * command will clear it (with a delay of one command). - *c : clear by read + * a : according to the card current state. + * b : always related to the previous command. reception of a valid + * command will clear it (with a delay of one command). + * c : clear by read */ -#define R5_COM_CRC_ERROR (1u << 15)/* er, b */ -#define R5_ILLEGAL_COMMAND (1u << 14)/* er, b */ -#define R5_IO_CURRENT_STATE_MASK (3u << 12)/* s, b */ -#define R5_IO_CURRENT_STATE(x) (((x) & R5_IO_CURRENT_STATE_MASK) >> 12) -#define R5_ERROR (1u << 11)/* erx, c */ -#define R5_FUNCTION_NUMBER (1u << 9)/* er, c */ -#define R5_OUT_OF_RANGE (1u << 8)/* er, c */ +#define R5_COM_CRC_ERROR (1u << 15) /* er, b */ +#define R5_ILLEGAL_COMMAND (1u << 14) /* er, b */ +#define R5_IO_CURRENT_STATE_MASK (3u << 12) /* s, b */ +#define R5_IO_CURRENT_STATE(x) (((x) & R5_IO_CURRENT_STATE_MASK) >> 12) +#define R5_ERROR (1u << 11) /* erx, c */ +#define R5_FUNCTION_NUMBER (1u << 9) /* er, c */ +#define R5_OUT_OF_RANGE (1u << 8) /* er, c */ + struct mmc_data { size_t len; /* size of the data */ size_t xfer_len; @@ -227,7 +228,7 @@ struct mmc_request { #define SD_SEND_RELATIVE_ADDR 3 #define MMC_SET_DSR 4 #define MMC_SLEEP_AWAKE 5 -#define IO_SEND_OP_COND 5 +#define IO_SEND_OP_COND 5 #define MMC_SWITCH_FUNC 6 #define MMC_SWITCH_FUNC_CMDS 0 #define MMC_SWITCH_FUNC_SET 1 @@ -318,30 +319,30 @@ struct mmc_request { /* Class 9: I/O cards (sd) */ #define SD_IO_RW_DIRECT 52 /* CMD52 arguments */ -#define SD_ARG_CMD52_READ (0<<31) -#define SD_ARG_CMD52_WRITE (1<<31) -#define SD_ARG_CMD52_FUNC_SHIFT 28 -#define SD_ARG_CMD52_FUNC_MASK 0x7 -#define SD_ARG_CMD52_EXCHANGE (1<<27) -#define SD_ARG_CMD52_REG_SHIFT 9 -#define SD_ARG_CMD52_REG_MASK 0x1ffff -#define SD_ARG_CMD52_DATA_SHIFT 0 -#define SD_ARG_CMD52_DATA_MASK 0xff -#define SD_R5_DATA(resp) ((resp)[0] & 0xff) +#define SD_ARG_CMD52_READ (0 << 31) +#define SD_ARG_CMD52_WRITE (1 << 31) +#define SD_ARG_CMD52_FUNC_SHIFT 28 +#define SD_ARG_CMD52_FUNC_MASK 0x7 +#define SD_ARG_CMD52_EXCHANGE (1 << 27) +#define SD_ARG_CMD52_REG_SHIFT 9 +#define SD_ARG_CMD52_REG_MASK 0x1ffff +#define SD_ARG_CMD52_DATA_SHIFT 0 +#define SD_ARG_CMD52_DATA_MASK 0xff +#define SD_R5_DATA(resp) ((resp)[0] & 0xff) #define SD_IO_RW_EXTENDED 53 /* CMD53 arguments */ -#define SD_ARG_CMD53_READ (0<<31) -#define SD_ARG_CMD53_WRITE (1<<31) -#define SD_ARG_CMD53_FUNC_SHIFT 28 -#define SD_ARG_CMD53_FUNC_MASK 0x7 -#define SD_ARG_CMD53_BLOCK_MODE (1<<27) -#define SD_ARG_CMD53_INCREMENT (1<<26) -#define SD_ARG_CMD53_REG_SHIFT 9 -#define SD_ARG_CMD53_REG_MASK 0x1ffff -#define SD_ARG_CMD53_LENGTH_SHIFT 0 -#define SD_ARG_CMD53_LENGTH_MASK 0x1ff -#define SD_ARG_CMD53_LENGTH_MAX 64 /* XXX should be 511? */ +#define SD_ARG_CMD53_READ (0 << 31) +#define SD_ARG_CMD53_WRITE (1 << 31) +#define SD_ARG_CMD53_FUNC_SHIFT 28 +#define SD_ARG_CMD53_FUNC_MASK 0x7 +#define SD_ARG_CMD53_BLOCK_MODE (1 << 27) +#define SD_ARG_CMD53_INCREMENT (1 << 26) +#define SD_ARG_CMD53_REG_SHIFT 9 +#define SD_ARG_CMD53_REG_MASK 0x1ffff +#define SD_ARG_CMD53_LENGTH_SHIFT 0 +#define SD_ARG_CMD53_LENGTH_MASK 0x1ff +#define SD_ARG_CMD53_LENGTH_MAX 64 /* XXX should be 511? */ /* Class 10: Switch function commands */ #define SD_SWITCH_FUNC 6 @@ -364,6 +365,8 @@ struct mmc_request { /* * EXT_CSD fields */ +#define EXT_CSD_FLUSH_CACHE 32 /* W/E */ +#define EXT_CSD_CACHE_CTRL 33 /* R/W/E */ #define EXT_CSD_EXT_PART_ATTR 52 /* R/W, 2 bytes */ #define EXT_CSD_ENH_START_ADDR 136 /* R/W, 4 bytes */ #define EXT_CSD_ENH_SIZE_MULT 140 /* R/W, 3 bytes */ @@ -397,12 +400,19 @@ struct mmc_request { #define EXT_CSD_PWR_CL_200_360 237 /* RO */ #define EXT_CSD_PWR_CL_52_195_DDR 238 /* RO */ #define EXT_CSD_PWR_CL_52_360_DDR 239 /* RO */ +#define EXT_CSD_CACHE_FLUSH_POLICY 249 /* RO */ #define EXT_CSD_GEN_CMD6_TIME 248 /* RO */ +#define EXT_CSD_CACHE_SIZE 249 /* RO, 4 bytes */ #define EXT_CSD_PWR_CL_200_360_DDR 253 /* RO */ /* * EXT_CSD field definitions */ +#define EXT_CSD_FLUSH_CACHE_FLUSH 0x01 +#define EXT_CSD_FLUSH_CACHE_BARRIER 0x02 + +#define EXT_CSD_CACHE_CTRL_CACHE_EN 0x01 + #define EXT_CSD_EXT_PART_ATTR_DEFAULT 0x0 #define EXT_CSD_EXT_PART_ATTR_SYSTEMCODE 0x1 #define EXT_CSD_EXT_PART_ATTR_NPERSISTENT 0x2 @@ -482,6 +492,8 @@ struct mmc_request { #define EXT_CSD_SEC_FEATURE_SUPPORT_GB_CL_EN 0x10 #define EXT_CSD_SEC_FEATURE_SUPPORT_SANITIZE 0x40 +#define EXT_CSD_CACHE_FLUSH_POLICY_FIFO 0x01 + /* * Vendor specific EXT_CSD fields */ @@ -533,50 +545,50 @@ struct mmc_request { /* * SDIO Direct & Extended I/O */ -#define SD_IO_RW_WR (1u << 31) -#define SD_IO_RW_FUNC(x) (((x) & 0x7) << 28) -#define SD_IO_RW_RAW (1u << 27) -#define SD_IO_RW_INCR (1u << 26) -#define SD_IO_RW_ADR(x) (((x) & 0x1FFFF) << 9) -#define SD_IO_RW_DAT(x) (((x) & 0xFF) << 0) -#define SD_IO_RW_LEN(x) (((x) & 0xFF) << 0) +#define SD_IO_RW_WR (1u << 31) +#define SD_IO_RW_FUNC(x) (((x) & 0x7) << 28) +#define SD_IO_RW_RAW (1u << 27) +#define SD_IO_RW_INCR (1u << 26) +#define SD_IO_RW_ADR(x) (((x) & 0x1FFFF) << 9) +#define SD_IO_RW_DAT(x) (((x) & 0xFF) << 0) +#define SD_IO_RW_LEN(x) (((x) & 0xFF) << 0) -#define SD_IOE_RW_LEN(x) (((x) & 0x1FF) << 0) -#define SD_IOE_RW_BLK (1u << 27) +#define SD_IOE_RW_LEN(x) (((x) & 0x1FF) << 0) +#define SD_IOE_RW_BLK (1u << 27) /* Card Common Control Registers (CCCR) */ -#define SD_IO_CCCR_START 0x00000 -#define SD_IO_CCCR_SIZE 0x100 -#define SD_IO_CCCR_FN_ENABLE 0x02 -#define SD_IO_CCCR_FN_READY 0x03 -#define SD_IO_CCCR_INT_ENABLE 0x04 -#define SD_IO_CCCR_INT_PENDING 0x05 -#define SD_IO_CCCR_CTL 0x06 -#define CCCR_CTL_RES (1<<3) -#define SD_IO_CCCR_BUS_WIDTH 0x07 -#define CCCR_BUS_WIDTH_4 (1<<1) -#define CCCR_BUS_WIDTH_1 (1<<0) -#define SD_IO_CCCR_CARDCAP 0x08 -#define SD_IO_CCCR_CISPTR 0x09 /* XXX 9-10, 10-11, or 9-12 */ +#define SD_IO_CCCR_START 0x00000 +#define SD_IO_CCCR_SIZE 0x100 +#define SD_IO_CCCR_FN_ENABLE 0x02 +#define SD_IO_CCCR_FN_READY 0x03 +#define SD_IO_CCCR_INT_ENABLE 0x04 +#define SD_IO_CCCR_INT_PENDING 0x05 +#define SD_IO_CCCR_CTL 0x06 +#define CCCR_CTL_RES (1 << 3) +#define SD_IO_CCCR_BUS_WIDTH 0x07 +#define CCCR_BUS_WIDTH_4 (1 << 1) +#define CCCR_BUS_WIDTH_1 (1 << 0) +#define SD_IO_CCCR_CARDCAP 0x08 +#define SD_IO_CCCR_CISPTR 0x09 /* XXX 9-10, 10-11, or 9-12 */ /* Function Basic Registers (FBR) */ -#define SD_IO_FBR_START 0x00100 -#define SD_IO_FBR_SIZE 0x00700 +#define SD_IO_FBR_START 0x00100 +#define SD_IO_FBR_SIZE 0x00700 /* Card Information Structure (CIS) */ -#define SD_IO_CIS_START 0x01000 -#define SD_IO_CIS_SIZE 0x17000 +#define SD_IO_CIS_START 0x01000 +#define SD_IO_CIS_SIZE 0x17000 /* CIS tuple codes (based on PC Card 16) */ -#define SD_IO_CISTPL_VERS_1 0x15 -#define SD_IO_CISTPL_MANFID 0x20 -#define SD_IO_CISTPL_FUNCID 0x21 -#define SD_IO_CISTPL_FUNCE 0x22 -#define SD_IO_CISTPL_END 0xff +#define SD_IO_CISTPL_VERS_1 0x15 +#define SD_IO_CISTPL_MANFID 0x20 +#define SD_IO_CISTPL_FUNCID 0x21 +#define SD_IO_CISTPL_FUNCE 0x22 +#define SD_IO_CISTPL_END 0xff /* CISTPL_FUNCID codes */ /* OpenBSD incorrectly defines 0x0c as FUNCTION_WLAN */ -/* #define SDMMC_FUNCTION_WLAN 0x0c */ +/* #define SDMMC_FUNCTION_WLAN 0x0c */ /* OCR bits */ diff --git a/freebsd/sys/dev/mmc/mmcsd.c b/freebsd/sys/dev/mmc/mmcsd.c index 94e6e73f..263da55d 100644 --- a/freebsd/sys/dev/mmc/mmcsd.c +++ b/freebsd/sys/dev/mmc/mmcsd.c @@ -71,7 +71,9 @@ __FBSDID("$FreeBSD$"); #include <sys/malloc.h> #include <sys/module.h> #include <sys/mutex.h> +#include <sys/priv.h> #include <sys/slicer.h> +#include <sys/sysctl.h> #include <sys/time.h> #include <geom/geom.h> @@ -142,6 +144,8 @@ struct mmcsd_softc { uint32_t flags; #define MMCSD_INAND_CMD38 0x0001 #define MMCSD_USE_TRIM 0x0002 +#define MMCSD_FLUSH_CACHE 0x0004 +#define MMCSD_DIRTY 0x0008 uint32_t cmd6_time; /* Generic switch timeout [us] */ uint32_t part_time; /* Partition switch timeout [us] */ off_t enh_base; /* Enhanced user data area slice base ... */ @@ -166,12 +170,19 @@ static const char * const errmsg[] = "NO MEMORY" }; +static SYSCTL_NODE(_hw, OID_AUTO, mmcsd, CTLFLAG_RD, NULL, "mmcsd driver"); + +static int mmcsd_cache = 1; +SYSCTL_INT(_hw_mmcsd, OID_AUTO, cache, CTLFLAG_RDTUN, &mmcsd_cache, 0, + "Device R/W cache enabled if present"); + #define LOG_PPS 5 /* Log no more than 5 errors per second. */ /* bus entry points */ static int mmcsd_attach(device_t dev); static int mmcsd_detach(device_t dev); static int mmcsd_probe(device_t dev); +static int mmcsd_shutdown(device_t dev); #ifndef __rtems__ /* disk routines */ @@ -181,7 +192,6 @@ static int mmcsd_dump(void *arg, void *virtual, vm_offset_t physical, static int mmcsd_getattr(struct bio *); static int mmcsd_ioctl_disk(struct disk *disk, u_long cmd, void *data, int fflag, struct thread *td); -static int mmcsd_open(struct disk *dp); static void mmcsd_strategy(struct bio *bp); static void mmcsd_task(void *arg); #endif /* __rtems__ */ @@ -197,8 +207,12 @@ static int mmcsd_bus_bit_width(device_t dev); static daddr_t mmcsd_delete(struct mmcsd_part *part, struct bio *bp); #endif /* __rtems__ */ static const char *mmcsd_errmsg(int e); +#ifndef __rtems__ +static int mmcsd_flush_cache(struct mmcsd_softc *sc); +#endif /* __rtems__ */ +static const char *mmcsd_errmsg(int e); static int mmcsd_ioctl(struct mmcsd_part *part, u_long cmd, void *data, - int fflag); + int fflag, struct thread *td); static int mmcsd_ioctl_cmd(struct mmcsd_part *part, struct mmc_ioc_cmd *mic, int fflag); static uintmax_t mmcsd_pretty_size(off_t size, char *unit); @@ -523,6 +537,31 @@ mmcsd_attach(device_t dev) rev = ext_csd[EXT_CSD_REV]; /* + * With revision 1.5 (MMC v4.5, EXT_CSD_REV == 6) and later, take + * advantage of the device R/W cache if present and useage is not + * disabled. + */ + if (rev >= 6 && mmcsd_cache != 0) { + size = ext_csd[EXT_CSD_CACHE_SIZE] | + ext_csd[EXT_CSD_CACHE_SIZE + 1] << 8 | + ext_csd[EXT_CSD_CACHE_SIZE + 2] << 16 | + ext_csd[EXT_CSD_CACHE_SIZE + 3] << 24; + if (bootverbose) + device_printf(dev, "cache size %juKB\n", size); + if (size > 0) { + MMCBUS_ACQUIRE_BUS(mmcbus, dev); + err = mmc_switch(mmcbus, dev, sc->rca, + EXT_CSD_CMD_SET_NORMAL, EXT_CSD_CACHE_CTRL, + EXT_CSD_CACHE_CTRL_CACHE_EN, sc->cmd6_time, true); + MMCBUS_RELEASE_BUS(mmcbus, dev); + if (err != MMC_ERR_NONE) + device_printf(dev, "failed to enable cache\n"); + else + sc->flags |= MMCSD_FLUSH_CACHE; + } + } + + /* * Ignore user-creatable enhanced user data area and general purpose * partitions partitions as long as partitioning hasn't been finished. */ @@ -734,7 +773,6 @@ mmcsd_add_part(struct mmcsd_softc *sc, u_int type, const char *name, u_int cnt, #ifndef __rtems__ d = part->disk = disk_alloc(); - d->d_open = mmcsd_open; d->d_close = mmcsd_close; d->d_strategy = mmcsd_strategy; d->d_ioctl = mmcsd_ioctl_disk; @@ -748,6 +786,8 @@ mmcsd_add_part(struct mmcsd_softc *sc, u_int type, const char *name, u_int cnt, d->d_stripesize = sc->erase_sector * d->d_sectorsize; d->d_unit = cnt; d->d_flags = DISKFLAG_CANDELETE; + if ((sc->flags & MMCSD_FLUSH_CACHE) != 0) + d->d_flags |= DISKFLAG_CANFLUSHCACHE; d->d_delmaxsize = mmc_get_erase_sector(dev) * d->d_sectorsize; strlcpy(d->d_ident, mmc_get_card_sn_string(dev), sizeof(d->d_ident)); @@ -908,6 +948,22 @@ mmcsd_detach(device_t dev) free(part, M_DEVBUF); } } + if (mmcsd_flush_cache(sc) != MMC_ERR_NONE) + device_printf(dev, "failed to flush cache\n"); +#else /* __rtems__ */ + BSD_PANIC("FIXME"); +#endif /* __rtems__ */ + return (0); +} + +static int +mmcsd_shutdown(device_t dev) +{ +#ifndef __rtems__ + struct mmcsd_softc *sc = device_get_softc(dev); + + if (mmcsd_flush_cache(sc) != MMC_ERR_NONE) + device_printf(dev, "failed to flush cache\n"); #else /* __rtems__ */ BSD_PANIC("FIXME"); #endif /* __rtems__ */ @@ -947,6 +1003,8 @@ mmcsd_suspend(device_t dev) MMCSD_IOCTL_UNLOCK(part); } } + if (mmcsd_flush_cache(sc) != MMC_ERR_NONE) + device_printf(dev, "failed to flush cache\n"); #else /* __rtems__ */ BSD_PANIC("FIXME"); #endif /* __rtems__ */ @@ -989,16 +1047,15 @@ mmcsd_resume(device_t dev) #ifndef __rtems__ static int -mmcsd_open(struct disk *dp __unused) -{ - - return (0); -} - -static int -mmcsd_close(struct disk *dp __unused) +mmcsd_close(struct disk *dp) { + struct mmcsd_softc *sc; + if ((dp->d_flags & DISKFLAG_OPEN) != 0) { + sc = ((struct mmcsd_part *)dp->d_drv1)->sc; + if (mmcsd_flush_cache(sc) != MMC_ERR_NONE) + device_printf(sc->dev, "failed to flush cache\n"); + } return (0); } @@ -1022,24 +1079,25 @@ mmcsd_strategy(struct bio *bp) static int mmcsd_ioctl_rpmb(struct cdev *dev, u_long cmd, caddr_t data, - int fflag, struct thread *td __unused) + int fflag, struct thread *td) { - return (mmcsd_ioctl(dev->si_drv1, cmd, data, fflag)); + return (mmcsd_ioctl(dev->si_drv1, cmd, data, fflag, td)); } #ifndef __rtems__ static int mmcsd_ioctl_disk(struct disk *disk, u_long cmd, void *data, int fflag, - struct thread *td __unused) + struct thread *td) { - return (mmcsd_ioctl(disk->d_drv1, cmd, data, fflag)); + return (mmcsd_ioctl(disk->d_drv1, cmd, data, fflag, td)); } #endif /* __rtems__ */ static int -mmcsd_ioctl(struct mmcsd_part *part, u_long cmd, void *data, int fflag) +mmcsd_ioctl(struct mmcsd_part *part, u_long cmd, void *data, int fflag, + struct thread *td) { struct mmc_ioc_cmd *mic; struct mmc_ioc_multi_cmd *mimc; @@ -1049,6 +1107,10 @@ mmcsd_ioctl(struct mmcsd_part *part, u_long cmd, void *data, int fflag) if ((fflag & FREAD) == 0) return (EBADF); + err = priv_check(td, PRIV_DRIVER); + if (err != 0) + return (err); + err = 0; switch (cmd) { case MMC_IOC_CMD: @@ -1190,6 +1252,8 @@ mmcsd_ioctl_cmd(struct mmcsd_part *part, struct mmc_ioc_cmd *mic, int fflag) if (err != MMC_ERR_NONE) goto switch_back; } + if (mic->write_flag != 0) + sc->flags |= MMCSD_DIRTY; if (mic->is_acmd != 0) (void)mmc_wait_for_app_cmd(mmcbus, dev, rca, &cmd, 0); else @@ -1405,6 +1469,7 @@ mmcsd_rw(struct mmcsd_part *part, struct bio *bp) else cmd.opcode = MMC_READ_SINGLE_BLOCK; } else { + sc->flags |= MMCSD_DIRTY; if (numblocks > 1) cmd.opcode = MMC_WRITE_MULTIPLE_BLOCK; else @@ -1590,13 +1655,18 @@ mmcsd_dump(void *arg, void *virtual, vm_offset_t physical, off_t offset, device_t dev, mmcbus; int err; - /* length zero is special and really means flush buffers to media */ - if (!length) - return (0); - disk = arg; part = disk->d_drv1; sc = part->sc; + + /* length zero is special and really means flush buffers to media */ + if (length == 0) { + err = mmcsd_flush_cache(sc); + if (err != MMC_ERR_NONE) + return (EIO); + return (0); + } + dev = sc->dev; mmcbus = sc->mmcbus; @@ -1646,6 +1716,14 @@ mmcsd_task(void *arg) "mmcsd disk jobqueue", 0); } while (bp == NULL); MMCSD_DISK_UNLOCK(part); + if (__predict_false(bp->bio_cmd == BIO_FLUSH)) { + if (mmcsd_flush_cache(sc) != MMC_ERR_NONE) { + bp->bio_error = EIO; + bp->bio_flags |= BIO_ERROR; + } + biodone(bp); + continue; + } if (bp->bio_cmd != BIO_READ && part->ro) { bp->bio_error = EROFS; bp->bio_resid = bp->bio_bcount; @@ -1704,10 +1782,37 @@ mmcsd_bus_bit_width(device_t dev) return (8); } +#ifndef __rtems__ +static int +mmcsd_flush_cache(struct mmcsd_softc *sc) +{ + device_t dev, mmcbus; + int err; + + if ((sc->flags & MMCSD_FLUSH_CACHE) == 0) + return (MMC_ERR_NONE); + + dev = sc->dev; + mmcbus = sc->mmcbus; + MMCBUS_ACQUIRE_BUS(mmcbus, dev); + if ((sc->flags & MMCSD_DIRTY) == 0) { + MMCBUS_RELEASE_BUS(mmcbus, dev); + return (MMC_ERR_NONE); + } + err = mmc_switch(mmcbus, dev, sc->rca, EXT_CSD_CMD_SET_NORMAL, + EXT_CSD_FLUSH_CACHE, EXT_CSD_FLUSH_CACHE_FLUSH, 60 * 1000, true); + if (err == MMC_ERR_NONE) + sc->flags &= ~MMCSD_DIRTY; + MMCBUS_RELEASE_BUS(mmcbus, dev); + return (err); +} +#endif /* __rtems__ */ + static device_method_t mmcsd_methods[] = { DEVMETHOD(device_probe, mmcsd_probe), DEVMETHOD(device_attach, mmcsd_attach), DEVMETHOD(device_detach, mmcsd_detach), + DEVMETHOD(device_shutdown, mmcsd_shutdown), DEVMETHOD(device_suspend, mmcsd_suspend), DEVMETHOD(device_resume, mmcsd_resume), DEVMETHOD_END diff --git a/freebsd/sys/dev/nvme/nvme.h b/freebsd/sys/dev/nvme/nvme.h index b51b4a97..169f22d1 100644 --- a/freebsd/sys/dev/nvme/nvme.h +++ b/freebsd/sys/dev/nvme/nvme.h @@ -115,7 +115,7 @@ #define NVME_CMD_FUSE_SHIFT (8) #define NVME_CMD_FUSE_MASK (0x3) -#define NVME_CMD_SET_OPC(opc) (htole16(((opc) & NVME_CMD_OPC_MASK) << NVME_CMD_OPC_SHIFT)) +#define NVME_CMD_SET_OPC(opc) (htole16(((uint16_t)(opc) & NVME_CMD_OPC_MASK) << NVME_CMD_OPC_SHIFT)) #define NVME_STATUS_P_SHIFT (0) #define NVME_STATUS_P_MASK (0x1) @@ -1091,6 +1091,12 @@ struct nvme_firmware_page { _Static_assert(sizeof(struct nvme_firmware_page) == 512, "bad size for nvme_firmware_page"); +struct nvme_ns_list { + uint32_t ns[1024]; +} __packed __aligned(4); + +_Static_assert(sizeof(struct nvme_ns_list) == 4096, "bad size for nvme_ns_list"); + struct intel_log_temp_stats { uint64_t current; @@ -1470,6 +1476,15 @@ void nvme_firmware_page_swapbytes(struct nvme_firmware_page *s) } static inline +void nvme_ns_list_swapbytes(struct nvme_ns_list *s) +{ + int i; + + for (i = 0; i < 1024; i++) + s->ns[i] = le32toh(s->ns[i]); +} + +static inline void intel_log_temp_stats_swapbytes(struct intel_log_temp_stats *s) { diff --git a/freebsd/sys/dev/ofw/ofw_bus_subr.c b/freebsd/sys/dev/ofw/ofw_bus_subr.c index 8406988f..5038bb03 100644 --- a/freebsd/sys/dev/ofw/ofw_bus_subr.c +++ b/freebsd/sys/dev/ofw/ofw_bus_subr.c @@ -59,12 +59,12 @@ ofw_bus_gen_setup_devinfo(struct ofw_bus_devinfo *obd, phandle_t node) if (obd == NULL) return (ENOMEM); /* The 'name' property is considered mandatory. */ - if ((OF_getprop_alloc(node, "name", 1, (void **)&obd->obd_name)) == -1) + if ((OF_getprop_alloc(node, "name", (void **)&obd->obd_name)) == -1) return (EINVAL); - OF_getprop_alloc(node, "compatible", 1, (void **)&obd->obd_compat); - OF_getprop_alloc(node, "device_type", 1, (void **)&obd->obd_type); - OF_getprop_alloc(node, "model", 1, (void **)&obd->obd_model); - OF_getprop_alloc(node, "status", 1, (void **)&obd->obd_status); + OF_getprop_alloc(node, "compatible", (void **)&obd->obd_compat); + OF_getprop_alloc(node, "device_type", (void **)&obd->obd_type); + OF_getprop_alloc(node, "model", (void **)&obd->obd_model); + OF_getprop_alloc(node, "status", (void **)&obd->obd_status); obd->obd_node = node; return (0); } @@ -321,10 +321,10 @@ ofw_bus_setup_iinfo(phandle_t node, struct ofw_bus_iinfo *ii, int intrsz) addrc = 2; ii->opi_addrc = addrc * sizeof(pcell_t); - ii->opi_imapsz = OF_getencprop_alloc(node, "interrupt-map", 1, + ii->opi_imapsz = OF_getencprop_alloc(node, "interrupt-map", (void **)&ii->opi_imap); if (ii->opi_imapsz > 0) { - msksz = OF_getencprop_alloc(node, "interrupt-map-mask", 1, + msksz = OF_getencprop_alloc(node, "interrupt-map-mask", (void **)&ii->opi_imapmsk); /* * Failure to get the mask is ignored; a full mask is used @@ -451,7 +451,8 @@ ofw_bus_msimap(phandle_t node, uint16_t pci_rid, phandle_t *msi_parent, int err, i; /* TODO: This should be OF_searchprop_alloc if we had it */ - len = OF_getencprop_alloc(node, "msi-map", sizeof(*map), (void **)&map); + len = OF_getencprop_alloc_multi(node, "msi-map", sizeof(*map), + (void **)&map); if (len < 0) { if (msi_parent != NULL) { *msi_parent = 0; @@ -491,9 +492,9 @@ ofw_bus_msimap(phandle_t node, uint16_t pci_rid, phandle_t *msi_parent, return (err); } -int -ofw_bus_reg_to_rl(device_t dev, phandle_t node, pcell_t acells, pcell_t scells, - struct resource_list *rl) +static int +ofw_bus_reg_to_rl_helper(device_t dev, phandle_t node, pcell_t acells, pcell_t scells, + struct resource_list *rl, const char *reg_source) { uint64_t phys, size; ssize_t i, j, rid, nreg, ret; @@ -504,11 +505,12 @@ ofw_bus_reg_to_rl(device_t dev, phandle_t node, pcell_t acells, pcell_t scells, * This may be just redundant when having ofw_bus_devinfo * but makes this routine independent of it. */ - ret = OF_getprop_alloc(node, "name", sizeof(*name), (void **)&name); + ret = OF_getprop_alloc(node, "name", (void **)&name); if (ret == -1) name = NULL; - ret = OF_getencprop_alloc(node, "reg", sizeof(*reg), (void **)®); + ret = OF_getencprop_alloc_multi(node, reg_source, sizeof(*reg), + (void **)®); nreg = (ret == -1) ? 0 : ret; if (nreg % (acells + scells) != 0) { @@ -539,6 +541,23 @@ ofw_bus_reg_to_rl(device_t dev, phandle_t node, pcell_t acells, pcell_t scells, return (0); } +int +ofw_bus_reg_to_rl(device_t dev, phandle_t node, pcell_t acells, pcell_t scells, + struct resource_list *rl) +{ + + return (ofw_bus_reg_to_rl_helper(dev, node, acells, scells, rl, "reg")); +} + +int +ofw_bus_assigned_addresses_to_rl(device_t dev, phandle_t node, pcell_t acells, + pcell_t scells, struct resource_list *rl) +{ + + return (ofw_bus_reg_to_rl_helper(dev, node, acells, scells, + rl, "assigned-addresses")); +} + /* * Get interrupt parent for given node. * Returns 0 if interrupt parent doesn't exist. @@ -569,7 +588,7 @@ ofw_bus_intr_to_rl(device_t dev, phandle_t node, int err, i, irqnum, nintr, rid; boolean_t extended; - nintr = OF_getencprop_alloc(node, "interrupts", sizeof(*intr), + nintr = OF_getencprop_alloc_multi(node, "interrupts", sizeof(*intr), (void **)&intr); if (nintr > 0) { iparent = ofw_bus_find_iparent(node); @@ -592,7 +611,7 @@ ofw_bus_intr_to_rl(device_t dev, phandle_t node, } extended = false; } else { - nintr = OF_getencprop_alloc(node, "interrupts-extended", + nintr = OF_getencprop_alloc_multi(node, "interrupts-extended", sizeof(*intr), (void **)&intr); if (nintr <= 0) return (0); @@ -635,7 +654,7 @@ ofw_bus_intr_by_rid(device_t dev, phandle_t node, int wanted_rid, int err, i, nintr, rid; boolean_t extended; - nintr = OF_getencprop_alloc(node, "interrupts", sizeof(*intr), + nintr = OF_getencprop_alloc_multi(node, "interrupts", sizeof(*intr), (void **)&intr); if (nintr > 0) { iparent = ofw_bus_find_iparent(node); @@ -658,7 +677,7 @@ ofw_bus_intr_by_rid(device_t dev, phandle_t node, int wanted_rid, } extended = false; } else { - nintr = OF_getencprop_alloc(node, "interrupts-extended", + nintr = OF_getencprop_alloc_multi(node, "interrupts-extended", sizeof(*intr), (void **)&intr); if (nintr <= 0) return (ESRCH); @@ -705,7 +724,7 @@ ofw_bus_find_child(phandle_t start, const char *child_name) phandle_t child; for (child = OF_child(start); child != 0; child = OF_peer(child)) { - ret = OF_getprop_alloc(child, "name", sizeof(*name), (void **)&name); + ret = OF_getprop_alloc(child, "name", (void **)&name); if (ret == -1) continue; if (strcmp(name, child_name) == 0) { @@ -806,7 +825,7 @@ ofw_bus_parse_xref_list_internal(phandle_t node, const char *list_name, int rv, i, j, nelems, cnt; elems = NULL; - nelems = OF_getencprop_alloc(node, list_name, sizeof(*elems), + nelems = OF_getencprop_alloc_multi(node, list_name, sizeof(*elems), (void **)&elems); if (nelems <= 0) return (ENOENT); @@ -901,7 +920,7 @@ ofw_bus_find_string_index(phandle_t node, const char *list_name, int rv, i, cnt, nelems; elems = NULL; - nelems = OF_getprop_alloc(node, list_name, 1, (void **)&elems); + nelems = OF_getprop_alloc(node, list_name, (void **)&elems); if (nelems <= 0) return (ENOENT); @@ -932,7 +951,7 @@ ofw_bus_string_list_to_array(phandle_t node, const char *list_name, int i, cnt, nelems, len; elems = NULL; - nelems = OF_getprop_alloc(node, list_name, 1, (void **)&elems); + nelems = OF_getprop_alloc(node, list_name, (void **)&elems); if (nelems <= 0) return (nelems); diff --git a/freebsd/sys/dev/ofw/ofw_bus_subr.h b/freebsd/sys/dev/ofw/ofw_bus_subr.h index 04f5a75b..7bf66a10 100644 --- a/freebsd/sys/dev/ofw/ofw_bus_subr.h +++ b/freebsd/sys/dev/ofw/ofw_bus_subr.h @@ -95,6 +95,8 @@ int ofw_bus_msimap(phandle_t, uint16_t, phandle_t *, uint32_t *); /* Routines for parsing device-tree data into resource lists. */ int ofw_bus_reg_to_rl(device_t, phandle_t, pcell_t, pcell_t, struct resource_list *); +int ofw_bus_assigned_addresses_to_rl(device_t, phandle_t, pcell_t, pcell_t, + struct resource_list *); int ofw_bus_intr_to_rl(device_t, phandle_t, struct resource_list *, int *); int ofw_bus_intr_by_rid(device_t, phandle_t, int, phandle_t *, int *, pcell_t **); diff --git a/freebsd/sys/dev/ofw/ofw_fdt.c b/freebsd/sys/dev/ofw/ofw_fdt.c index 8878b5c9..b1bbadee 100644 --- a/freebsd/sys/dev/ofw/ofw_fdt.c +++ b/freebsd/sys/dev/ofw/ofw_fdt.c @@ -134,7 +134,7 @@ sysctl_register_fdt_oid(void *arg) CTLTYPE_OPAQUE | CTLFLAG_RD, NULL, 0, sysctl_handle_dtb, "", "Device Tree Blob"); } -SYSINIT(dtb_oid, SI_SUB_KMEM, SI_ORDER_ANY, sysctl_register_fdt_oid, 0); +SYSINIT(dtb_oid, SI_SUB_KMEM, SI_ORDER_ANY, sysctl_register_fdt_oid, NULL); static int ofw_fdt_init(ofw_t ofw, void *data) diff --git a/freebsd/sys/dev/ofw/openfirm.c b/freebsd/sys/dev/ofw/openfirm.c index 406e8dd6..9cc7dbdc 100644 --- a/freebsd/sys/dev/ofw/openfirm.c +++ b/freebsd/sys/dev/ofw/openfirm.c @@ -442,7 +442,7 @@ OF_searchprop(phandle_t node, const char *propname, void *buf, size_t len) } ssize_t -OF_searchencprop(phandle_t node, const char *propname, void *buf, size_t len) +OF_searchencprop(phandle_t node, const char *propname, pcell_t *buf, size_t len) { ssize_t rv; @@ -454,11 +454,35 @@ OF_searchencprop(phandle_t node, const char *propname, void *buf, size_t len) /* * Store the value of a property of a package into newly allocated memory + * (using the M_OFWPROP malloc pool and M_WAITOK). + */ +ssize_t +OF_getprop_alloc(phandle_t package, const char *propname, void **buf) +{ + int len; + + *buf = NULL; + if ((len = OF_getproplen(package, propname)) == -1) + return (-1); + + if (len > 0) { + *buf = malloc(len, M_OFWPROP, M_WAITOK); + if (OF_getprop(package, propname, *buf, len) == -1) { + free(*buf, M_OFWPROP); + *buf = NULL; + return (-1); + } + } + return (len); +} + +/* + * Store the value of a property of a package into newly allocated memory * (using the M_OFWPROP malloc pool and M_WAITOK). elsz is the size of a * single element, the number of elements is return in number. */ ssize_t -OF_getprop_alloc(phandle_t package, const char *propname, int elsz, void **buf) +OF_getprop_alloc_multi(phandle_t package, const char *propname, int elsz, void **buf) { int len; @@ -467,30 +491,41 @@ OF_getprop_alloc(phandle_t package, const char *propname, int elsz, void **buf) len % elsz != 0) return (-1); - *buf = malloc(len, M_OFWPROP, M_WAITOK); - if (OF_getprop(package, propname, *buf, len) == -1) { - free(*buf, M_OFWPROP); - *buf = NULL; - return (-1); + if (len > 0) { + *buf = malloc(len, M_OFWPROP, M_WAITOK); + if (OF_getprop(package, propname, *buf, len) == -1) { + free(*buf, M_OFWPROP); + *buf = NULL; + return (-1); + } } return (len / elsz); } ssize_t -OF_getencprop_alloc(phandle_t package, const char *name, int elsz, void **buf) +OF_getencprop_alloc(phandle_t package, const char *name, void **buf) +{ + ssize_t ret; + + ret = OF_getencprop_alloc_multi(package, name, sizeof(pcell_t), + buf); + if (ret < 0) + return (ret); + else + return (ret * sizeof(pcell_t)); +} + +ssize_t +OF_getencprop_alloc_multi(phandle_t package, const char *name, int elsz, + void **buf) { ssize_t retval; pcell_t *cell; int i; - retval = OF_getprop_alloc(package, name, elsz, buf); + retval = OF_getprop_alloc_multi(package, name, elsz, buf); if (retval == -1) return (-1); - if (retval * elsz % 4 != 0) { - free(*buf, M_OFWPROP); - *buf = NULL; - return (-1); - } cell = *buf; for (i = 0; i < retval * elsz / 4; i++) diff --git a/freebsd/sys/dev/ofw/openfirm.h b/freebsd/sys/dev/ofw/openfirm.h index e1701164..f043197a 100644 --- a/freebsd/sys/dev/ofw/openfirm.h +++ b/freebsd/sys/dev/ofw/openfirm.h @@ -114,10 +114,14 @@ int OF_hasprop(phandle_t node, const char *propname); ssize_t OF_searchprop(phandle_t node, const char *propname, void *buf, size_t len); ssize_t OF_searchencprop(phandle_t node, const char *propname, - void *buf, size_t len); + pcell_t *buf, size_t len); ssize_t OF_getprop_alloc(phandle_t node, const char *propname, + void **buf); +ssize_t OF_getprop_alloc_multi(phandle_t node, const char *propname, int elsz, void **buf); ssize_t OF_getencprop_alloc(phandle_t node, const char *propname, + void **buf); +ssize_t OF_getencprop_alloc_multi(phandle_t node, const char *propname, int elsz, void **buf); void OF_prop_free(void *buf); int OF_nextprop(phandle_t node, const char *propname, char *buf, diff --git a/freebsd/sys/dev/pci/pci_pci.c b/freebsd/sys/dev/pci/pci_pci.c index 20364cf9..ec52e1fb 100644 --- a/freebsd/sys/dev/pci/pci_pci.c +++ b/freebsd/sys/dev/pci/pci_pci.c @@ -2547,6 +2547,22 @@ pcib_enable_ari(struct pcib_softc *sc, uint32_t pcie_pos) int pcib_maxslots(device_t dev) { +#if !defined(__amd64__) && !defined(__i386__) + uint32_t pcie_pos; + uint16_t val; + + /* + * If this is a PCIe rootport or downstream switch port, there's only + * one slot permitted. + */ + if (pci_find_cap(dev, PCIY_EXPRESS, &pcie_pos) == 0) { + val = pci_read_config(dev, pcie_pos + PCIER_FLAGS, 2); + val &= PCIEM_FLAGS_TYPE; + if (val == PCIEM_TYPE_ROOT_PORT || + val == PCIEM_TYPE_DOWNSTREAM_PORT) + return (0); + } +#endif return (PCI_SLOTMAX); } @@ -2560,7 +2576,7 @@ pcib_ari_maxslots(device_t dev) if (sc->flags & PCIB_ENABLE_ARI) return (PCIE_ARI_SLOTMAX); else - return (PCI_SLOTMAX); + return (pcib_maxslots(dev)); } static int diff --git a/freebsd/sys/dev/pci/pci_user.c b/freebsd/sys/dev/pci/pci_user.c index e6297bd7..c9d500a8 100644 --- a/freebsd/sys/dev/pci/pci_user.c +++ b/freebsd/sys/dev/pci/pci_user.c @@ -32,7 +32,6 @@ __FBSDID("$FreeBSD$"); #include <rtems/bsd/local/opt_bus.h> /* XXX trim includes */ -#include <rtems/bsd/local/opt_compat.h> #include <sys/param.h> #include <sys/systm.h> @@ -68,8 +67,6 @@ __FBSDID("$FreeBSD$"); static d_open_t pci_open; static d_close_t pci_close; -static int pci_conf_match(struct pci_match_conf *matches, int num_matches, - struct pci_conf *match_buf); static d_ioctl_t pci_ioctl; struct cdevsw pcicdev = { @@ -109,7 +106,7 @@ pci_close(struct cdev *dev, int flag, int devtype, struct thread *td) * This function returns 1 on failure, 0 on success. */ static int -pci_conf_match(struct pci_match_conf *matches, int num_matches, +pci_conf_match_native(struct pci_match_conf *matches, int num_matches, struct pci_conf *match_buf) { int i; @@ -278,9 +275,6 @@ struct pci_conf_io32 { #define PCIOCREAD_OLD _IOWR('p', 2, struct pci_io_old) #define PCIOCWRITE_OLD _IOWR('p', 3, struct pci_io_old) -static int pci_conf_match_old(struct pci_match_conf_old *matches, - int num_matches, struct pci_conf *match_buf); - static int pci_conf_match_old(struct pci_match_conf_old *matches, int num_matches, struct pci_conf *match_buf) @@ -408,7 +402,44 @@ pci_conf_match_old32(struct pci_match_conf_old32 *matches, int num_matches, return (1); } #endif /* COMPAT_FREEBSD32 */ -#endif /* PRE7_COMPAT */ +#endif /* !PRE7_COMPAT */ + +union pci_conf_union { + struct pci_conf pc; +#ifdef PRE7_COMPAT + struct pci_conf_old pco; +#ifdef COMPAT_FREEBSD32 + struct pci_conf_old32 pco32; +#endif +#endif +}; + +static int +pci_conf_match(u_long cmd, struct pci_match_conf *matches, int num_matches, + struct pci_conf *match_buf) +{ + + switch (cmd) { + case PCIOCGETCONF: + return (pci_conf_match_native( + (struct pci_match_conf *)matches, num_matches, match_buf)); +#ifdef PRE7_COMPAT + case PCIOCGETCONF_OLD: + return (pci_conf_match_old( + (struct pci_match_conf_old *)matches, num_matches, + match_buf)); +#ifdef COMPAT_FREEBSD32 + case PCIOCGETCONF_OLD32: + return (pci_conf_match_old32( + (struct pci_match_conf_old32 *)matches, num_matches, + match_buf)); +#endif +#endif + default: + /* programmer error */ + return (0); + } +} static int pci_list_vpd(device_t dev, struct pci_list_vpd_io *lvio) @@ -493,11 +524,184 @@ pci_list_vpd(device_t dev, struct pci_list_vpd_io *lvio) return (0); } +static size_t +pci_match_conf_size(u_long cmd) +{ + + switch (cmd) { + case PCIOCGETCONF: + return (sizeof(struct pci_match_conf)); +#ifdef PRE7_COMPAT + case PCIOCGETCONF_OLD: + return (sizeof(struct pci_match_conf_old)); +#ifdef COMPAT_FREEBSD32 + case PCIOCGETCONF_OLD32: + return (sizeof(struct pci_match_conf_old32)); +#endif +#endif + default: + /* programmer error */ + return (0); + } +} + +static size_t +pci_conf_size(u_long cmd) +{ + + switch (cmd) { + case PCIOCGETCONF: + return (sizeof(struct pci_conf)); +#ifdef PRE7_COMPAT + case PCIOCGETCONF_OLD: + return (sizeof(struct pci_conf_old)); +#ifdef COMPAT_FREEBSD32 + case PCIOCGETCONF_OLD32: + return (sizeof(struct pci_conf_old32)); +#endif +#endif + default: + /* programmer error */ + return (0); + } +} + +static void +pci_conf_io_init(struct pci_conf_io *cio, caddr_t data, u_long cmd) +{ +#if defined(PRE7_COMPAT) && defined(COMPAT_FREEBSD32) + struct pci_conf_io32 *cio32; +#endif + + switch (cmd) { + case PCIOCGETCONF: +#ifdef PRE7_COMPAT + case PCIOCGETCONF_OLD: +#endif + *cio = *(struct pci_conf_io *)data; + return; + +#if defined(PRE7_COMPAT) && defined(COMPAT_FREEBSD32) + case PCIOCGETCONF_OLD32: + cio32 = (struct pci_conf_io32 *)data; + cio->pat_buf_len = cio32->pat_buf_len; + cio->num_patterns = cio32->num_patterns; + cio->patterns = (void *)(uintptr_t)cio32->patterns; + cio->match_buf_len = cio32->match_buf_len; + cio->num_matches = cio32->num_matches; + cio->matches = (void *)(uintptr_t)cio32->matches; + cio->offset = cio32->offset; + cio->generation = cio32->generation; + cio->status = cio32->status; + return; +#endif + + default: + /* programmer error */ + return; + } +} + +static void +pci_conf_io_update_data(const struct pci_conf_io *cio, caddr_t data, + u_long cmd) +{ + struct pci_conf_io *d_cio; +#if defined(PRE7_COMPAT) && defined(COMPAT_FREEBSD32) + struct pci_conf_io32 *cio32; +#endif + + switch (cmd) { + case PCIOCGETCONF: +#ifdef PRE7_COMPAT + case PCIOCGETCONF_OLD: +#endif + d_cio = (struct pci_conf_io *)data; + d_cio->status = cio->status; + d_cio->generation = cio->generation; + d_cio->offset = cio->offset; + d_cio->num_matches = cio->num_matches; + return; + +#if defined(PRE7_COMPAT) && defined(COMPAT_FREEBSD32) + case PCIOCGETCONF_OLD32: + cio32 = (struct pci_conf_io32 *)data; + + cio32->status = cio->status; + cio32->generation = cio->generation; + cio32->offset = cio->offset; + cio32->num_matches = cio->num_matches; + return; +#endif + + default: + /* programmer error */ + return; + } +} + +static void +pci_conf_for_copyout(const struct pci_conf *pcp, union pci_conf_union *pcup, + u_long cmd) +{ + + memset(pcup, 0, sizeof(*pcup)); + + switch (cmd) { + case PCIOCGETCONF: + pcup->pc = *pcp; + return; + +#ifdef PRE7_COMPAT +#ifdef COMPAT_FREEBSD32 + case PCIOCGETCONF_OLD32: + pcup->pco32.pc_sel.pc_bus = pcp->pc_sel.pc_bus; + pcup->pco32.pc_sel.pc_dev = pcp->pc_sel.pc_dev; + pcup->pco32.pc_sel.pc_func = pcp->pc_sel.pc_func; + pcup->pco32.pc_hdr = pcp->pc_hdr; + pcup->pco32.pc_subvendor = pcp->pc_subvendor; + pcup->pco32.pc_subdevice = pcp->pc_subdevice; + pcup->pco32.pc_vendor = pcp->pc_vendor; + pcup->pco32.pc_device = pcp->pc_device; + pcup->pco32.pc_class = pcp->pc_class; + pcup->pco32.pc_subclass = pcp->pc_subclass; + pcup->pco32.pc_progif = pcp->pc_progif; + pcup->pco32.pc_revid = pcp->pc_revid; + strlcpy(pcup->pco32.pd_name, pcp->pd_name, + sizeof(pcup->pco32.pd_name)); + pcup->pco32.pd_unit = (uint32_t)pcp->pd_unit; + return; + +#endif /* COMPAT_FREEBSD32 */ + case PCIOCGETCONF_OLD: + pcup->pco.pc_sel.pc_bus = pcp->pc_sel.pc_bus; + pcup->pco.pc_sel.pc_dev = pcp->pc_sel.pc_dev; + pcup->pco.pc_sel.pc_func = pcp->pc_sel.pc_func; + pcup->pco.pc_hdr = pcp->pc_hdr; + pcup->pco.pc_subvendor = pcp->pc_subvendor; + pcup->pco.pc_subdevice = pcp->pc_subdevice; + pcup->pco.pc_vendor = pcp->pc_vendor; + pcup->pco.pc_device = pcp->pc_device; + pcup->pco.pc_class = pcp->pc_class; + pcup->pco.pc_subclass = pcp->pc_subclass; + pcup->pco.pc_progif = pcp->pc_progif; + pcup->pco.pc_revid = pcp->pc_revid; + strlcpy(pcup->pco.pd_name, pcp->pd_name, + sizeof(pcup->pco.pd_name)); + pcup->pco.pd_unit = pcp->pd_unit; + return; +#endif /* PRE7_COMPAT */ + + default: + /* programmer error */ + return; + } +} + static int pci_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *td) { device_t pcidev; - void *confdata; const char *name; struct devlist *devlist_head; struct pci_conf_io *cio = NULL; @@ -507,31 +711,25 @@ pci_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *t struct pci_list_vpd_io *lvio; struct pci_match_conf *pattern_buf; struct pci_map *pm; - size_t confsz, iolen, pbufsz; + size_t confsz, iolen; int error, ionum, i, num_patterns; + union pci_conf_union pcu; #ifdef PRE7_COMPAT -#ifdef COMPAT_FREEBSD32 - struct pci_conf_io32 *cio32 = NULL; - struct pci_conf_old32 conf_old32; - struct pci_match_conf_old32 *pattern_buf_old32 = NULL; -#endif - struct pci_conf_old conf_old; struct pci_io iodata; struct pci_io_old *io_old; - struct pci_match_conf_old *pattern_buf_old = NULL; io_old = NULL; #endif if (!(flag & FWRITE)) { switch (cmd) { + case PCIOCGETCONF: #ifdef PRE7_COMPAT + case PCIOCGETCONF_OLD: #ifdef COMPAT_FREEBSD32 case PCIOCGETCONF_OLD32: #endif - case PCIOCGETCONF_OLD: #endif - case PCIOCGETCONF: case PCIOCGETBAR: case PCIOCLISTVPD: break; @@ -540,39 +738,18 @@ pci_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *t } } - switch (cmd) { -#ifdef PRE7_COMPAT -#ifdef COMPAT_FREEBSD32 - case PCIOCGETCONF_OLD32: - cio32 = (struct pci_conf_io32 *)data; - cio = malloc(sizeof(struct pci_conf_io), M_TEMP, M_WAITOK); - cio->pat_buf_len = cio32->pat_buf_len; - cio->num_patterns = cio32->num_patterns; - cio->patterns = (void *)(uintptr_t)cio32->patterns; - cio->match_buf_len = cio32->match_buf_len; - cio->num_matches = cio32->num_matches; - cio->matches = (void *)(uintptr_t)cio32->matches; - cio->offset = cio32->offset; - cio->generation = cio32->generation; - cio->status = cio32->status; - cio32->num_matches = 0; - break; -#endif - case PCIOCGETCONF_OLD: -#endif - case PCIOCGETCONF: - cio = (struct pci_conf_io *)data; - } switch (cmd) { + case PCIOCGETCONF: #ifdef PRE7_COMPAT + case PCIOCGETCONF_OLD: #ifdef COMPAT_FREEBSD32 case PCIOCGETCONF_OLD32: #endif - case PCIOCGETCONF_OLD: #endif - case PCIOCGETCONF: - + cio = malloc(sizeof(struct pci_conf_io), M_TEMP, + M_WAITOK | M_ZERO); + pci_conf_io_init(cio, data, cmd); pattern_buf = NULL; num_patterns = 0; dinfo = NULL; @@ -611,17 +788,7 @@ pci_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *t * multiple of sizeof(struct pci_conf) in case the user * didn't specify a multiple of that size. */ -#ifdef PRE7_COMPAT -#ifdef COMPAT_FREEBSD32 - if (cmd == PCIOCGETCONF_OLD32) - confsz = sizeof(struct pci_conf_old32); - else -#endif - if (cmd == PCIOCGETCONF_OLD) - confsz = sizeof(struct pci_conf_old); - else -#endif - confsz = sizeof(struct pci_conf); + confsz = pci_conf_size(cmd); iolen = min(cio->match_buf_len - (cio->match_buf_len % confsz), pci_numdevs * confsz); @@ -650,18 +817,8 @@ pci_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *t * it's far more likely to just catch folks that * updated their kernel but not their userland. */ -#ifdef PRE7_COMPAT -#ifdef COMPAT_FREEBSD32 - if (cmd == PCIOCGETCONF_OLD32) - pbufsz = sizeof(struct pci_match_conf_old32); - else -#endif - if (cmd == PCIOCGETCONF_OLD) - pbufsz = sizeof(struct pci_match_conf_old); - else -#endif - pbufsz = sizeof(struct pci_match_conf); - if (cio->num_patterns * pbufsz != cio->pat_buf_len) { + if (cio->num_patterns * pci_match_conf_size(cmd) != + cio->pat_buf_len) { /* The user made a mistake, return an error. */ cio->status = PCI_GETCONF_ERROR; error = EINVAL; @@ -671,28 +828,10 @@ pci_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *t /* * Allocate a buffer to hold the patterns. */ -#ifdef PRE7_COMPAT -#ifdef COMPAT_FREEBSD32 - if (cmd == PCIOCGETCONF_OLD32) { - pattern_buf_old32 = malloc(cio->pat_buf_len, - M_TEMP, M_WAITOK); - error = copyin(cio->patterns, - pattern_buf_old32, cio->pat_buf_len); - } else -#endif /* COMPAT_FREEBSD32 */ - if (cmd == PCIOCGETCONF_OLD) { - pattern_buf_old = malloc(cio->pat_buf_len, - M_TEMP, M_WAITOK); - error = copyin(cio->patterns, - pattern_buf_old, cio->pat_buf_len); - } else -#endif /* PRE7_COMPAT */ - { - pattern_buf = malloc(cio->pat_buf_len, M_TEMP, - M_WAITOK); - error = copyin(cio->patterns, pattern_buf, - cio->pat_buf_len); - } + pattern_buf = malloc(cio->pat_buf_len, M_TEMP, + M_WAITOK); + error = copyin(cio->patterns, pattern_buf, + cio->pat_buf_len); if (error != 0) { error = EINVAL; goto getconfexit; @@ -735,27 +874,9 @@ pci_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *t dinfo->conf.pd_unit = 0; } -#ifdef PRE7_COMPAT - if ( -#ifdef COMPAT_FREEBSD32 - (cmd == PCIOCGETCONF_OLD32 && - (pattern_buf_old32 == NULL || - pci_conf_match_old32(pattern_buf_old32, - num_patterns, &dinfo->conf) == 0)) || -#endif - (cmd == PCIOCGETCONF_OLD && - (pattern_buf_old == NULL || - pci_conf_match_old(pattern_buf_old, num_patterns, - &dinfo->conf) == 0)) || - (cmd == PCIOCGETCONF && - (pattern_buf == NULL || - pci_conf_match(pattern_buf, num_patterns, - &dinfo->conf) == 0))) { -#else if (pattern_buf == NULL || - pci_conf_match(pattern_buf, num_patterns, + pci_conf_match(cmd, pattern_buf, num_patterns, &dinfo->conf) == 0) { -#endif /* * If we've filled up the user's buffer, * break out at this point. Since we've @@ -769,79 +890,8 @@ pci_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *t break; } -#ifdef PRE7_COMPAT -#ifdef COMPAT_FREEBSD32 - if (cmd == PCIOCGETCONF_OLD32) { - memset(&conf_old32, 0, - sizeof(conf_old32)); - conf_old32.pc_sel.pc_bus = - dinfo->conf.pc_sel.pc_bus; - conf_old32.pc_sel.pc_dev = - dinfo->conf.pc_sel.pc_dev; - conf_old32.pc_sel.pc_func = - dinfo->conf.pc_sel.pc_func; - conf_old32.pc_hdr = dinfo->conf.pc_hdr; - conf_old32.pc_subvendor = - dinfo->conf.pc_subvendor; - conf_old32.pc_subdevice = - dinfo->conf.pc_subdevice; - conf_old32.pc_vendor = - dinfo->conf.pc_vendor; - conf_old32.pc_device = - dinfo->conf.pc_device; - conf_old32.pc_class = - dinfo->conf.pc_class; - conf_old32.pc_subclass = - dinfo->conf.pc_subclass; - conf_old32.pc_progif = - dinfo->conf.pc_progif; - conf_old32.pc_revid = - dinfo->conf.pc_revid; - strncpy(conf_old32.pd_name, - dinfo->conf.pd_name, - sizeof(conf_old32.pd_name)); - conf_old32.pd_name[PCI_MAXNAMELEN] = 0; - conf_old32.pd_unit = - (uint32_t)dinfo->conf.pd_unit; - confdata = &conf_old32; - } else -#endif /* COMPAT_FREEBSD32 */ - if (cmd == PCIOCGETCONF_OLD) { - memset(&conf_old, 0, sizeof(conf_old)); - conf_old.pc_sel.pc_bus = - dinfo->conf.pc_sel.pc_bus; - conf_old.pc_sel.pc_dev = - dinfo->conf.pc_sel.pc_dev; - conf_old.pc_sel.pc_func = - dinfo->conf.pc_sel.pc_func; - conf_old.pc_hdr = dinfo->conf.pc_hdr; - conf_old.pc_subvendor = - dinfo->conf.pc_subvendor; - conf_old.pc_subdevice = - dinfo->conf.pc_subdevice; - conf_old.pc_vendor = - dinfo->conf.pc_vendor; - conf_old.pc_device = - dinfo->conf.pc_device; - conf_old.pc_class = - dinfo->conf.pc_class; - conf_old.pc_subclass = - dinfo->conf.pc_subclass; - conf_old.pc_progif = - dinfo->conf.pc_progif; - conf_old.pc_revid = - dinfo->conf.pc_revid; - strncpy(conf_old.pd_name, - dinfo->conf.pd_name, - sizeof(conf_old.pd_name)); - conf_old.pd_name[PCI_MAXNAMELEN] = 0; - conf_old.pd_unit = - dinfo->conf.pd_unit; - confdata = &conf_old; - } else -#endif /* PRE7_COMPAT */ - confdata = &dinfo->conf; - error = copyout(confdata, + pci_conf_for_copyout(&dinfo->conf, &pcu, cmd); + error = copyout(&pcu, (caddr_t)cio->matches + confsz * cio->num_matches, confsz); if (error) @@ -874,23 +924,9 @@ pci_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *t cio->status = PCI_GETCONF_MORE_DEVS; getconfexit: -#ifdef PRE7_COMPAT -#ifdef COMPAT_FREEBSD32 - if (cmd == PCIOCGETCONF_OLD32) { - cio32->status = cio->status; - cio32->generation = cio->generation; - cio32->offset = cio->offset; - cio32->num_matches = cio->num_matches; - free(cio, M_TEMP); - } - if (pattern_buf_old32 != NULL) - free(pattern_buf_old32, M_TEMP); -#endif - if (pattern_buf_old != NULL) - free(pattern_buf_old, M_TEMP); -#endif - if (pattern_buf != NULL) - free(pattern_buf, M_TEMP); + pci_conf_io_update_data(cio, data, cmd); + free(cio, M_TEMP); + free(pattern_buf, M_TEMP); break; diff --git a/freebsd/sys/dev/re/if_re.c b/freebsd/sys/dev/re/if_re.c index c8bb6db7..5aa24349 100644 --- a/freebsd/sys/dev/re/if_re.c +++ b/freebsd/sys/dev/re/if_re.c @@ -141,6 +141,8 @@ __FBSDID("$FreeBSD$"); #include <net/bpf.h> +#include <netinet/netdump/netdump.h> + #include <machine/bus.h> #include <machine/resource.h> #include <sys/bus.h> @@ -281,6 +283,7 @@ static void re_tick (void *); static void re_int_task (void *, int); static void re_start (struct ifnet *); static void re_start_locked (struct ifnet *); +static void re_start_tx (struct rl_softc *); static int re_ioctl (struct ifnet *, u_long, caddr_t); static void re_init (void *); static void re_init_locked (struct rl_softc *); @@ -309,6 +312,8 @@ static void re_setwol (struct rl_softc *); static void re_clrwol (struct rl_softc *); static void re_set_linkspeed (struct rl_softc *); +NETDUMP_DEFINE(re); + #ifdef DEV_NETMAP /* see ixgbe.c for details */ #include <dev/netmap/if_re_netmap.h> MODULE_DEPEND(re, netmap, 1, 1, 1); @@ -682,7 +687,7 @@ re_set_rxmode(struct rl_softc *sc) } if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; h = ether_crc32_be(LLADDR((struct sockaddr_dl *) @@ -1739,8 +1744,11 @@ re_attach(device_t dev) if (error) { device_printf(dev, "couldn't set up irq\n"); ether_ifdetach(ifp); + goto fail; } + NETDUMP_SET(ifp, re); + fail: if (error) re_detach(dev); @@ -2935,7 +2943,7 @@ re_start_locked(struct ifnet *ifp) #ifdef DEV_NETMAP /* XXX is this necessary ? */ if (ifp->if_capenable & IFCAP_NETMAP) { - struct netmap_kring *kring = &NA(ifp)->tx_rings[0]; + struct netmap_kring *kring = NA(ifp)->tx_rings[0]; if (sc->rl_ldata.rl_tx_prodidx != kring->nr_hwcur) { /* kick the tx unit */ CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START); @@ -2983,8 +2991,14 @@ re_start_locked(struct ifnet *ifp) return; } - /* Flush the TX descriptors */ + re_start_tx(sc); +} +static void +re_start_tx(struct rl_softc *sc) +{ + + /* Flush the TX descriptors */ bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag, sc->rl_ldata.rl_tx_list_map, BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD); @@ -4080,3 +4094,59 @@ sysctl_hw_re_int_mod(SYSCTL_HANDLER_ARGS) return (sysctl_int_range(oidp, arg1, arg2, req, RL_TIMER_MIN, RL_TIMER_MAX)); } + +#ifdef NETDUMP +static void +re_netdump_init(struct ifnet *ifp, int *nrxr, int *ncl, int *clsize) +{ + struct rl_softc *sc; + + sc = if_getsoftc(ifp); + RL_LOCK(sc); + *nrxr = sc->rl_ldata.rl_rx_desc_cnt; + *ncl = NETDUMP_MAX_IN_FLIGHT; + *clsize = (ifp->if_mtu > RL_MTU && + (sc->rl_flags & RL_FLAG_JUMBOV2) != 0) ? MJUM9BYTES : MCLBYTES; + RL_UNLOCK(sc); +} + +static void +re_netdump_event(struct ifnet *ifp __unused, enum netdump_ev event __unused) +{ +} + +static int +re_netdump_transmit(struct ifnet *ifp, struct mbuf *m) +{ + struct rl_softc *sc; + int error; + + sc = if_getsoftc(ifp); + if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != + IFF_DRV_RUNNING || (sc->rl_flags & RL_FLAG_LINK) == 0) + return (EBUSY); + + error = re_encap(sc, &m); + if (error == 0) + re_start_tx(sc); + return (error); +} + +static int +re_netdump_poll(struct ifnet *ifp, int count) +{ + struct rl_softc *sc; + int error; + + sc = if_getsoftc(ifp); + if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0 || + (sc->rl_flags & RL_FLAG_LINK) == 0) + return (EBUSY); + + re_txeof(sc); + error = re_rxeof(sc, NULL); + if (error != 0 && error != EAGAIN) + return (error); + return (0); +} +#endif /* NETDUMP */ diff --git a/freebsd/sys/dev/rtwn/if_rtwn_rx.c b/freebsd/sys/dev/rtwn/if_rtwn_rx.c index fb260596..512292ce 100644 --- a/freebsd/sys/dev/rtwn/if_rtwn_rx.c +++ b/freebsd/sys/dev/rtwn/if_rtwn_rx.c @@ -389,7 +389,7 @@ rtwn_set_multi(struct rtwn_softc *sc) TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) { ifp = vap->iv_ifp; if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { caddr_t dl; uint8_t pos; diff --git a/freebsd/sys/dev/tsec/if_tsec.c b/freebsd/sys/dev/tsec/if_tsec.c index e07c21ce..c18c950f 100644 --- a/freebsd/sys/dev/tsec/if_tsec.c +++ b/freebsd/sys/dev/tsec/if_tsec.c @@ -1943,7 +1943,7 @@ tsec_setup_multicast(struct tsec_softc *sc) } if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; diff --git a/freebsd/sys/dev/usb/input/ukbd.c b/freebsd/sys/dev/usb/input/ukbd.c index 5cb2636c..76fe76b6 100644 --- a/freebsd/sys/dev/usb/input/ukbd.c +++ b/freebsd/sys/dev/usb/input/ukbd.c @@ -41,7 +41,6 @@ __FBSDID("$FreeBSD$"); * HID spec: http://www.usb.org/developers/devclass_docs/HID1_11.pdf */ -#include <rtems/bsd/local/opt_compat.h> #include <rtems/bsd/local/opt_kbd.h> #include <rtems/bsd/local/opt_ukbd.h> #include <rtems/bsd/local/opt_evdev.h> diff --git a/freebsd/sys/dev/usb/input/usb_rdesc.h b/freebsd/sys/dev/usb/input/usb_rdesc.h index 00eea122..889381e8 100644 --- a/freebsd/sys/dev/usb/input/usb_rdesc.h +++ b/freebsd/sys/dev/usb/input/usb_rdesc.h @@ -276,3 +276,31 @@ 0x81, 0x01, /* INPUT (Constant) */\ 0xc0 /* END COLLECTION */\ +/* Fixed report descriptor for Super Nintendo gamepads */ +#define UHID_SNES_REPORT_DESCR(...) \ + 0x05, 0x01, /* Usage Page (Desktop), */\ + 0x09, 0x04, /* Usage (Joystik), */\ + 0xA1, 0x01, /* Collection (Application), */\ + 0xA1, 0x02, /* Collection (Logical), */\ + 0x14, /* Logical Minimum (0), */\ + 0x75, 0x08, /* Report Size (8), */\ + 0x95, 0x03, /* Report Count (3), */\ + 0x81, 0x01, /* Input (Constant), */\ + 0x26, 0xFF, 0x00, /* Logical Maximum (255), */\ + 0x95, 0x02, /* Report Count (2), */\ + 0x09, 0x30, /* Usage (X), */\ + 0x09, 0x31, /* Usage (Y), */\ + 0x81, 0x02, /* Input (Variable), */\ + 0x75, 0x01, /* Report Size (1), */\ + 0x95, 0x04, /* Report Count (4), */\ + 0x81, 0x01, /* Input (Constant), */\ + 0x25, 0x01, /* Logical Maximum (1), */\ + 0x95, 0x0A, /* Report Count (10), */\ + 0x05, 0x09, /* Usage Page (Button), */\ + 0x19, 0x01, /* Usage Minimum (01h), */\ + 0x29, 0x0A, /* Usage Maximum (0Ah), */\ + 0x81, 0x02, /* Input (Variable), */\ + 0x95, 0x0A, /* Report Count (10), */\ + 0x81, 0x01, /* Input (Constant), */\ + 0xC0, /* End Collection, */\ + 0xC0 /* End Collection */ diff --git a/freebsd/sys/dev/usb/net/if_aue.c b/freebsd/sys/dev/usb/net/if_aue.c index 69951b70..5454e2aa 100644 --- a/freebsd/sys/dev/usb/net/if_aue.c +++ b/freebsd/sys/dev/usb/net/if_aue.c @@ -557,7 +557,7 @@ aue_setmulti(struct usb_ether *ue) /* now program new ones */ if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; h = ether_crc32_le(LLADDR((struct sockaddr_dl *) diff --git a/freebsd/sys/dev/usb/net/if_axe.c b/freebsd/sys/dev/usb/net/if_axe.c index b77292ba..cf54e96e 100644 --- a/freebsd/sys/dev/usb/net/if_axe.c +++ b/freebsd/sys/dev/usb/net/if_axe.c @@ -504,7 +504,7 @@ axe_setmulti(struct usb_ether *ue) rxmode &= ~AXE_RXCMD_ALLMULTI; if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; diff --git a/freebsd/sys/dev/usb/net/if_axge.c b/freebsd/sys/dev/usb/net/if_axge.c index 575571e4..f8ed34ae 100644 --- a/freebsd/sys/dev/usb/net/if_axge.c +++ b/freebsd/sys/dev/usb/net/if_axge.c @@ -788,7 +788,7 @@ axge_rxfilter(struct usb_ether *ue) rxmode |= RCR_ACPT_MCAST; if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; h = ether_crc32_be(LLADDR((struct sockaddr_dl *) diff --git a/freebsd/sys/dev/usb/net/if_cue.c b/freebsd/sys/dev/usb/net/if_cue.c index 63846a01..f184fa24 100644 --- a/freebsd/sys/dev/usb/net/if_cue.c +++ b/freebsd/sys/dev/usb/net/if_cue.c @@ -330,7 +330,7 @@ cue_setmulti(struct usb_ether *ue) /* now program new ones */ if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; diff --git a/freebsd/sys/dev/usb/net/if_kue.c b/freebsd/sys/dev/usb/net/if_kue.c index 255a83bb..35fd3fcf 100644 --- a/freebsd/sys/dev/usb/net/if_kue.c +++ b/freebsd/sys/dev/usb/net/if_kue.c @@ -379,7 +379,7 @@ kue_setmulti(struct usb_ether *ue) sc->sc_rxfilt &= ~KUE_RXFILT_ALLMULTI; if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; diff --git a/freebsd/sys/dev/usb/net/if_mos.c b/freebsd/sys/dev/usb/net/if_mos.c index ad7bfcce..3dd0e5ef 100644 --- a/freebsd/sys/dev/usb/net/if_mos.c +++ b/freebsd/sys/dev/usb/net/if_mos.c @@ -608,7 +608,7 @@ mos_setmulti(struct usb_ether *ue) /* get all new ones */ if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) { allmulti = 1; continue; diff --git a/freebsd/sys/dev/usb/net/if_rue.c b/freebsd/sys/dev/usb/net/if_rue.c index 7b062755..810a98c8 100644 --- a/freebsd/sys/dev/usb/net/if_rue.c +++ b/freebsd/sys/dev/usb/net/if_rue.c @@ -502,7 +502,7 @@ rue_setmulti(struct usb_ether *ue) /* now program new ones */ if_maddr_rlock(ifp); - TAILQ_FOREACH (ifma, &ifp->if_multiaddrs, ifma_link) + CK_STAILQ_FOREACH (ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; diff --git a/freebsd/sys/dev/usb/net/if_smsc.c b/freebsd/sys/dev/usb/net/if_smsc.c index 88ecb618..87e181d8 100644 --- a/freebsd/sys/dev/usb/net/if_smsc.c +++ b/freebsd/sys/dev/usb/net/if_smsc.c @@ -454,7 +454,7 @@ smsc_miibus_readreg(device_t dev, int phy, int reg) goto done; } - addr = (phy << 11) | (reg << 6) | SMSC_MII_READ; + addr = (phy << 11) | (reg << 6) | SMSC_MII_READ | SMSC_MII_BUSY; smsc_write_reg(sc, SMSC_MII_ADDR, addr); if (smsc_wait_for_bits(sc, SMSC_MII_ADDR, SMSC_MII_BUSY) != 0) @@ -507,7 +507,7 @@ smsc_miibus_writereg(device_t dev, int phy, int reg, int val) val = htole32(val); smsc_write_reg(sc, SMSC_MII_DATA, val); - addr = (phy << 11) | (reg << 6) | SMSC_MII_WRITE; + addr = (phy << 11) | (reg << 6) | SMSC_MII_WRITE | SMSC_MII_BUSY; smsc_write_reg(sc, SMSC_MII_ADDR, addr); if (smsc_wait_for_bits(sc, SMSC_MII_ADDR, SMSC_MII_BUSY) != 0) @@ -716,14 +716,14 @@ smsc_setmulti(struct usb_ether *ue) /* Take the lock of the mac address list before hashing each of them */ if_maddr_rlock(ifp); - if (!TAILQ_EMPTY(&ifp->if_multiaddrs)) { + if (!CK_STAILQ_EMPTY(&ifp->if_multiaddrs)) { /* We are filtering on a set of address so calculate hashes of each * of the address and set the corresponding bits in the register. */ sc->sc_mac_csr |= SMSC_MAC_CSR_HPFILT; sc->sc_mac_csr &= ~(SMSC_MAC_CSR_PRMS | SMSC_MAC_CSR_MCPAS); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; @@ -1308,7 +1308,7 @@ smsc_phy_init(struct smsc_softc *sc) do { uether_pause(&sc->sc_ue, hz / 100); bmcr = smsc_miibus_readreg(sc->sc_ue.ue_dev, sc->sc_phyno, MII_BMCR); - } while ((bmcr & MII_BMCR) && ((ticks - start_ticks) < max_ticks)); + } while ((bmcr & BMCR_RESET) && ((ticks - start_ticks) < max_ticks)); if (((usb_ticks_t)(ticks - start_ticks)) >= max_ticks) { smsc_err_printf(sc, "PHY reset timed-out"); diff --git a/freebsd/sys/dev/usb/net/if_udav.c b/freebsd/sys/dev/usb/net/if_udav.c index 21019265..a4e683ac 100644 --- a/freebsd/sys/dev/usb/net/if_udav.c +++ b/freebsd/sys/dev/usb/net/if_udav.c @@ -524,7 +524,7 @@ udav_setmulti(struct usb_ether *ue) /* now program new ones */ if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; diff --git a/freebsd/sys/dev/usb/net/if_ure.c b/freebsd/sys/dev/usb/net/if_ure.c index 0e45a6c9..8a88feae 100644 --- a/freebsd/sys/dev/usb/net/if_ure.c +++ b/freebsd/sys/dev/usb/net/if_ure.c @@ -797,7 +797,7 @@ ure_rxfilter(struct usb_ether *ue) rxmode |= URE_RCR_AM; if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; h = ether_crc32_be(LLADDR((struct sockaddr_dl *) diff --git a/freebsd/sys/dev/usb/net/usb_ethernet.c b/freebsd/sys/dev/usb/net/usb_ethernet.c index 842e7987..9ce60eff 100644 --- a/freebsd/sys/dev/usb/net/usb_ethernet.c +++ b/freebsd/sys/dev/usb/net/usb_ethernet.c @@ -192,6 +192,17 @@ error: return (error); } +void +uether_ifattach_wait(struct usb_ether *ue) +{ + + UE_LOCK(ue); + usb_proc_mwait(&ue->ue_tq, + &ue->ue_sync_task[0].hdr, + &ue->ue_sync_task[1].hdr); + UE_UNLOCK(ue); +} + static void ue_attach_post_task(struct usb_proc_msg *_task) { diff --git a/freebsd/sys/dev/usb/net/usb_ethernet.h b/freebsd/sys/dev/usb/net/usb_ethernet.h index c7afc650..9839db16 100644 --- a/freebsd/sys/dev/usb/net/usb_ethernet.h +++ b/freebsd/sys/dev/usb/net/usb_ethernet.h @@ -113,6 +113,7 @@ struct ifnet *uether_getifp(struct usb_ether *); struct mii_data *uether_getmii(struct usb_ether *); void *uether_getsc(struct usb_ether *); int uether_ifattach(struct usb_ether *); +void uether_ifattach_wait(struct usb_ether *); void uether_ifdetach(struct usb_ether *); int uether_ifmedia_upd(struct ifnet *); void uether_init(void *); diff --git a/freebsd/sys/dev/usb/serial/uchcom.c b/freebsd/sys/dev/usb/serial/uchcom.c index 1e741ccd..24cb8433 100644 --- a/freebsd/sys/dev/usb/serial/uchcom.c +++ b/freebsd/sys/dev/usb/serial/uchcom.c @@ -124,11 +124,11 @@ SYSCTL_INT(_hw_usb_uchcom, OID_AUTO, debug, CTLFLAG_RWTUN, #define UCHCOM_REG_BPS_MOD 0x14 #define UCHCOM_REG_BPS_PAD 0x0F #define UCHCOM_REG_BREAK1 0x05 -#define UCHCOM_REG_BREAK2 0x18 #define UCHCOM_REG_LCR1 0x18 #define UCHCOM_REG_LCR2 0x25 #define UCHCOM_VER_20 0x20 +#define UCHCOM_VER_30 0x30 #define UCHCOM_BASE_UNKNOWN 0 #define UCHCOM_BPS_MOD_BASE 20000000 @@ -137,12 +137,14 @@ SYSCTL_INT(_hw_usb_uchcom, OID_AUTO, debug, CTLFLAG_RWTUN, #define UCHCOM_DTR_MASK 0x20 #define UCHCOM_RTS_MASK 0x40 -#define UCHCOM_BRK1_MASK 0x01 -#define UCHCOM_BRK2_MASK 0x40 +#define UCHCOM_BRK_MASK 0x01 #define UCHCOM_LCR1_MASK 0xAF #define UCHCOM_LCR2_MASK 0x07 -#define UCHCOM_LCR1_PARENB 0x80 +#define UCHCOM_LCR1_RX 0x80 +#define UCHCOM_LCR1_TX 0x40 +#define UCHCOM_LCR1_PARENB 0x08 +#define UCHCOM_LCR1_CS8 0x03 #define UCHCOM_LCR2_PAREVEN 0x07 #define UCHCOM_LCR2_PARODD 0x06 #define UCHCOM_LCR2_PARMARK 0x05 @@ -324,13 +326,17 @@ uchcom_attach(device_t dev) sc->sc_udev = uaa->device; - switch (uaa->info.bcdDevice) { - case UCHCOM_REV_CH340: + switch (uaa->info.idProduct) { + case USB_PRODUCT_WCH2_CH341SER: device_printf(dev, "CH340 detected\n"); break; - default: + case USB_PRODUCT_WCH2_CH341SER_2: device_printf(dev, "CH341 detected\n"); break; + default: + device_printf(dev, "New CH340/CH341 product 0x%04x detected\n", + uaa->info.idProduct); + break; } iface_index = UCHCOM_IFACE_INDEX; @@ -414,6 +420,8 @@ uchcom_ctrl_write(struct uchcom_softc *sc, uint8_t reqno, USETW(req.wIndex, index); USETW(req.wLength, 0); + DPRINTF("WR REQ 0x%02X VAL 0x%04X IDX 0x%04X\n", + reqno, value, index); ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, &req, NULL, 0, 1000); } @@ -430,6 +438,8 @@ uchcom_ctrl_read(struct uchcom_softc *sc, uint8_t reqno, USETW(req.wIndex, index); USETW(req.wLength, buflen); + DPRINTF("RD REQ 0x%02X VAL 0x%04X IDX 0x%04X LEN %d\n", + reqno, value, index, buflen); ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, &req, buf, USB_SHORT_XFER_OK, 1000); } @@ -504,6 +514,7 @@ static void uchcom_update_version(struct uchcom_softc *sc) { uchcom_get_version(sc, &sc->sc_version); + DPRINTF("Chip version: 0x%02x\n", sc->sc_version); } static void @@ -549,17 +560,17 @@ uchcom_cfg_set_break(struct ucom_softc *ucom, uint8_t onoff) uint8_t brk1; uint8_t brk2; - uchcom_read_reg(sc, UCHCOM_REG_BREAK1, &brk1, UCHCOM_REG_BREAK2, &brk2); + uchcom_read_reg(sc, UCHCOM_REG_BREAK1, &brk1, UCHCOM_REG_LCR1, &brk2); if (onoff) { /* on - clear bits */ - brk1 &= ~UCHCOM_BRK1_MASK; - brk2 &= ~UCHCOM_BRK2_MASK; + brk1 &= ~UCHCOM_BRK_MASK; + brk2 &= ~UCHCOM_LCR1_TX; } else { /* off - set bits */ - brk1 |= UCHCOM_BRK1_MASK; - brk2 |= UCHCOM_BRK2_MASK; + brk1 |= UCHCOM_BRK_MASK; + brk2 |= UCHCOM_LCR1_TX; } - uchcom_write_reg(sc, UCHCOM_REG_BREAK1, brk1, UCHCOM_REG_BREAK2, brk2); + uchcom_write_reg(sc, UCHCOM_REG_BREAK1, brk1, UCHCOM_REG_LCR1, brk2); } static int @@ -611,8 +622,12 @@ uchcom_set_baudrate(struct uchcom_softc *sc, uint32_t rate) if (uchcom_calc_divider_settings(&dv, rate)) return; + /* + * According to linux code we need to set bit 7 of UCHCOM_REG_BPS_PRE, + * otherwise the chip will buffer data. + */ uchcom_write_reg(sc, - UCHCOM_REG_BPS_PRE, dv.dv_prescaler, + UCHCOM_REG_BPS_PRE, dv.dv_prescaler | 0x80, UCHCOM_REG_BPS_DIV, dv.dv_div); uchcom_write_reg(sc, UCHCOM_REG_BPS_MOD, dv.dv_mod, @@ -678,6 +693,10 @@ uchcom_pre_param(struct ucom_softc *ucom, struct termios *t) default: return (EIO); } + if ((t->c_cflag & CSTOPB) != 0) + return (EIO); + if ((t->c_cflag & PARENB) != 0) + return (EIO); if (uchcom_calc_divider_settings(&dv, t->c_ospeed)) { return (EIO); @@ -690,11 +709,26 @@ uchcom_cfg_param(struct ucom_softc *ucom, struct termios *t) { struct uchcom_softc *sc = ucom->sc_parent; - uchcom_get_version(sc, 0); + uchcom_get_version(sc, NULL); uchcom_ctrl_write(sc, UCHCOM_REQ_RESET, 0, 0); uchcom_set_baudrate(sc, t->c_ospeed); - uchcom_read_reg(sc, 0x18, 0, 0x25, 0); - uchcom_write_reg(sc, 0x18, 0x50, 0x25, 0x00); + if (sc->sc_version < UCHCOM_VER_30) { + uchcom_read_reg(sc, UCHCOM_REG_LCR1, NULL, + UCHCOM_REG_LCR2, NULL); + uchcom_write_reg(sc, UCHCOM_REG_LCR1, 0x50, + UCHCOM_REG_LCR2, 0x00); + } else { + /* + * Set up line control: + * - enable transmit and receive + * - set 8n1 mode + * To do: support other sizes, parity, stop bits. + */ + uchcom_write_reg(sc, + UCHCOM_REG_LCR1, + UCHCOM_LCR1_RX | UCHCOM_LCR1_TX | UCHCOM_LCR1_CS8, + UCHCOM_REG_LCR2, 0x00); + } uchcom_update_status(sc); uchcom_ctrl_write(sc, UCHCOM_REQ_RESET, 0x501f, 0xd90a); uchcom_set_baudrate(sc, t->c_ospeed); diff --git a/freebsd/sys/dev/usb/serial/umodem.c b/freebsd/sys/dev/usb/serial/umodem.c index ac1e35c8..76c7dfe2 100644 --- a/freebsd/sys/dev/usb/serial/umodem.c +++ b/freebsd/sys/dev/usb/serial/umodem.c @@ -459,6 +459,8 @@ umodem_attach(device_t dev) mtx_unlock(&sc->sc_mtx); } + ucom_set_usb_mode(&sc->sc_super_ucom, uaa->usb_mode); + error = ucom_attach(&sc->sc_super_ucom, &sc->sc_ucom, 1, sc, &umodem_callback, &sc->sc_mtx); if (error) { diff --git a/freebsd/sys/dev/usb/serial/usb_serial.c b/freebsd/sys/dev/usb/serial/usb_serial.c index 46a18d63..a3f9b5de 100644 --- a/freebsd/sys/dev/usb/serial/usb_serial.c +++ b/freebsd/sys/dev/usb/serial/usb_serial.c @@ -109,6 +109,12 @@ static int ucom_pps_mode; SYSCTL_INT(_hw_usb_ucom, OID_AUTO, pps_mode, CTLFLAG_RWTUN, &ucom_pps_mode, 0, "pulse capture mode: 0/1/2=disabled/CTS/DCD; add 0x10 to invert"); + +static int ucom_device_mode_console = 1; + +SYSCTL_INT(_hw_usb_ucom, OID_AUTO, device_mode_console, CTLFLAG_RW, + &ucom_device_mode_console, 0, + "set to 1 to mark terminals as consoles when in device mode"); #endif /* __rtems__ */ #ifdef USB_DEBUG @@ -203,6 +209,7 @@ ucom_init(void *arg) } SYSINIT(ucom_init, SI_SUB_KLD - 1, SI_ORDER_ANY, ucom_init, NULL); +#ifndef __rtems__ static void ucom_uninit(void *arg) { @@ -218,6 +225,7 @@ ucom_uninit(void *arg) mtx_destroy(&ucom_mtx); } SYSUNINIT(ucom_uninit, SI_SUB_KLD - 3, SI_ORDER_ANY, ucom_uninit, NULL); +#endif /* __rtems__ */ /* * Mark a unit number (the X in cuaUX) as in use. @@ -294,7 +302,7 @@ ucom_attach(struct ucom_super_softc *ssc, struct ucom_softc *sc, } ssc->sc_subunits = subunits; ssc->sc_flag = UCOM_FLAG_ATTACHED | - UCOM_FLAG_FREE_UNIT; + UCOM_FLAG_FREE_UNIT | (ssc->sc_flag & UCOM_FLAG_DEVICE_MODE); if (callback->ucom_free == NULL) ssc->sc_flag |= UCOM_FLAG_WAIT_REFS; @@ -394,6 +402,24 @@ ucom_drain_all(void *arg) mtx_unlock(&ucom_mtx); } +static cn_probe_t ucom_cnprobe; +static cn_init_t ucom_cninit; +static cn_term_t ucom_cnterm; +static cn_getc_t ucom_cngetc; +static cn_putc_t ucom_cnputc; +static cn_grab_t ucom_cngrab; +static cn_ungrab_t ucom_cnungrab; + +const struct consdev_ops ucom_cnops = { + .cn_probe = ucom_cnprobe, + .cn_init = ucom_cninit, + .cn_term = ucom_cnterm, + .cn_getc = ucom_cngetc, + .cn_putc = ucom_cnputc, + .cn_grab = ucom_cngrab, + .cn_ungrab = ucom_cnungrab, +}; + static int ucom_attach_tty(struct ucom_super_softc *ssc, struct ucom_softc *sc) { @@ -458,6 +484,26 @@ ucom_attach_tty(struct ucom_super_softc *ssc, struct ucom_softc *sc) UCOM_MTX_UNLOCK(ucom_cons_softc); } +#ifndef __rtems__ + if ((ssc->sc_flag & UCOM_FLAG_DEVICE_MODE) != 0 && + ucom_device_mode_console > 0 && + ucom_cons_softc == NULL) { + struct consdev *cp; + + cp = malloc(sizeof(struct consdev), M_USBDEV, + M_WAITOK|M_ZERO); + cp->cn_ops = &ucom_cnops; + cp->cn_arg = NULL; + cp->cn_pri = CN_NORMAL; + strlcpy(cp->cn_name, "tty", sizeof(cp->cn_name)); + strlcat(cp->cn_name, buf, sizeof(cp->cn_name)); + + sc->sc_consdev = cp; + + cnadd(cp); + } +#endif /* __rtems__ */ + return (0); } @@ -468,6 +514,14 @@ ucom_detach_tty(struct ucom_super_softc *ssc, struct ucom_softc *sc) DPRINTF("sc = %p, tp = %p\n", sc, sc->sc_tty); +#ifndef __rtems__ + if (sc->sc_consdev != NULL) { + cnremove(sc->sc_consdev); + free(sc->sc_consdev, M_USBDEV); + sc->sc_consdev = NULL; + } +#endif /* __rtems__ */ + if (sc->sc_flag & UCOM_FLAG_CONSOLE) { UCOM_MTX_LOCK(ucom_cons_softc); ucom_close(ucom_cons_softc->sc_tty); @@ -541,6 +595,20 @@ ucom_set_pnpinfo_usb(struct ucom_super_softc *ssc, device_t dev) } } +void +ucom_set_usb_mode(struct ucom_super_softc *ssc, enum usb_hc_mode usb_mode) +{ + + switch (usb_mode) { + case USB_MODE_DEVICE: + ssc->sc_flag |= UCOM_FLAG_DEVICE_MODE; + break; + default: + ssc->sc_flag &= ~UCOM_FLAG_DEVICE_MODE; + break; + } +} + static void ucom_queue_command(struct ucom_softc *sc, usb_proc_callback_t *fn, struct termios *pt, @@ -1547,14 +1615,6 @@ ucom_free(void *xsc) mtx_unlock(&ucom_mtx); } -static cn_probe_t ucom_cnprobe; -static cn_init_t ucom_cninit; -static cn_term_t ucom_cnterm; -static cn_getc_t ucom_cngetc; -static cn_putc_t ucom_cnputc; -static cn_grab_t ucom_cngrab; -static cn_ungrab_t ucom_cnungrab; - CONSOLE_DRIVER(ucom); static void diff --git a/freebsd/sys/dev/usb/serial/usb_serial.h b/freebsd/sys/dev/usb/serial/usb_serial.h index 9a5e043e..2bcc388d 100644 --- a/freebsd/sys/dev/usb/serial/usb_serial.h +++ b/freebsd/sys/dev/usb/serial/usb_serial.h @@ -165,6 +165,9 @@ struct ucom_softc { const struct ucom_callback *sc_callback; struct ucom_super_softc *sc_super; struct tty *sc_tty; +#ifndef __rtems__ + struct consdev *sc_consdev; +#endif /* __rtems__ */ struct mtx *sc_mtx; void *sc_parent; int sc_subunit; @@ -183,6 +186,7 @@ struct ucom_softc { #define UCOM_FLAG_FREE_UNIT 0x0200 /* set if we must free the unit */ #define UCOM_FLAG_INWAKEUP 0x0400 /* set if we are in the tsw_inwakeup callback */ #define UCOM_FLAG_LSRTXIDLE 0x0800 /* set if sc_lsr bits ULSR_TSRE+TXRDY work */ +#define UCOM_FLAG_DEVICE_MODE 0x1000 /* set if we're an USB device, not a host */ uint8_t sc_lsr; uint8_t sc_msr; uint8_t sc_mcr; @@ -211,6 +215,7 @@ int ucom_attach(struct ucom_super_softc *, const struct ucom_callback *callback, struct mtx *); void ucom_detach(struct ucom_super_softc *, struct ucom_softc *); void ucom_set_pnpinfo_usb(struct ucom_super_softc *, device_t); +void ucom_set_usb_mode(struct ucom_super_softc *, enum usb_hc_mode); void ucom_status_change(struct ucom_softc *); uint8_t ucom_get_data(struct ucom_softc *, struct usb_page_cache *, uint32_t, uint32_t, uint32_t *); diff --git a/freebsd/sys/dev/usb/serial/uslcom.c b/freebsd/sys/dev/usb/serial/uslcom.c index 45835b82..4128802d 100644 --- a/freebsd/sys/dev/usb/serial/uslcom.c +++ b/freebsd/sys/dev/usb/serial/uslcom.c @@ -66,7 +66,7 @@ SYSCTL_INT(_hw_usb_uslcom, OID_AUTO, debug, CTLFLAG_RWTUN, &uslcom_debug, 0, "Debug level"); #endif -#define USLCOM_BULK_BUF_SIZE 1024 +#define USLCOM_BULK_BUF_SIZE 1024 #define USLCOM_CONFIG_INDEX 0 /* Request types */ @@ -75,13 +75,13 @@ SYSCTL_INT(_hw_usb_uslcom, OID_AUTO, debug, CTLFLAG_RWTUN, /* Request codes */ #define USLCOM_IFC_ENABLE 0x00 -#define USLCOM_SET_BAUDDIV 0x01 +#define USLCOM_SET_BAUDDIV 0x01 #define USLCOM_SET_LINE_CTL 0x03 #define USLCOM_SET_BREAK 0x05 #define USLCOM_SET_MHS 0x07 #define USLCOM_GET_MDMSTS 0x08 #define USLCOM_SET_FLOW 0x13 -#define USLCOM_SET_BAUDRATE 0x1e +#define USLCOM_SET_BAUDRATE 0x1e #define USLCOM_VENDOR_SPECIFIC 0xff /* USLCOM_IFC_ENABLE values */ @@ -89,7 +89,7 @@ SYSCTL_INT(_hw_usb_uslcom, OID_AUTO, debug, CTLFLAG_RWTUN, #define USLCOM_IFC_ENABLE_EN 0x01 /* USLCOM_SET_MHS/USLCOM_GET_MDMSTS values */ -#define USLCOM_MHS_DTR_ON 0x0001 +#define USLCOM_MHS_DTR_ON 0x0001 #define USLCOM_MHS_DTR_SET 0x0100 #define USLCOM_MHS_RTS_ON 0x0002 #define USLCOM_MHS_RTS_SET 0x0200 @@ -114,11 +114,11 @@ SYSCTL_INT(_hw_usb_uslcom, OID_AUTO, debug, CTLFLAG_RWTUN, #define USLCOM_SET_BREAK_ON 0x01 /* USLCOM_SET_FLOW values - 1st word */ -#define USLCOM_FLOW_DTR_ON 0x00000001 /* DTR static active */ -#define USLCOM_FLOW_CTS_HS 0x00000008 /* CTS handshake */ +#define USLCOM_FLOW_DTR_ON 0x00000001 /* DTR static active */ +#define USLCOM_FLOW_CTS_HS 0x00000008 /* CTS handshake */ /* USLCOM_SET_FLOW values - 2nd word */ -#define USLCOM_FLOW_RTS_ON 0x00000040 /* RTS static active */ -#define USLCOM_FLOW_RTS_HS 0x00000080 /* RTS handshake */ +#define USLCOM_FLOW_RTS_ON 0x00000040 /* RTS static active */ +#define USLCOM_FLOW_RTS_HS 0x00000080 /* RTS handshake */ /* USLCOM_VENDOR_SPECIFIC values */ #define USLCOM_GET_PARTNUM 0x370B @@ -148,10 +148,10 @@ struct uslcom_softc { struct usb_device *sc_udev; struct mtx sc_mtx; - uint8_t sc_msr; - uint8_t sc_lsr; - uint8_t sc_iface_no; - uint8_t sc_partnum; + uint8_t sc_msr; + uint8_t sc_lsr; + uint8_t sc_iface_no; + uint8_t sc_partnum; }; static device_probe_t uslcom_probe; @@ -163,18 +163,18 @@ static usb_callback_t uslcom_write_callback; static usb_callback_t uslcom_read_callback; static usb_callback_t uslcom_control_callback; -static void uslcom_free(struct ucom_softc *); -static void uslcom_open(struct ucom_softc *); -static void uslcom_close(struct ucom_softc *); +static void uslcom_free(struct ucom_softc *); static uint8_t uslcom_get_partnum(struct uslcom_softc *); -static void uslcom_set_dtr(struct ucom_softc *, uint8_t); -static void uslcom_set_rts(struct ucom_softc *, uint8_t); -static void uslcom_set_break(struct ucom_softc *, uint8_t); +static void uslcom_cfg_open(struct ucom_softc *); +static void uslcom_cfg_close(struct ucom_softc *); +static void uslcom_cfg_set_dtr(struct ucom_softc *, uint8_t); +static void uslcom_cfg_set_rts(struct ucom_softc *, uint8_t); +static void uslcom_cfg_set_break(struct ucom_softc *, uint8_t); +static void uslcom_cfg_param(struct ucom_softc *, struct termios *); +static void uslcom_cfg_get_status(struct ucom_softc *, uint8_t *, uint8_t *); static int uslcom_ioctl(struct ucom_softc *, uint32_t, caddr_t, int, - struct thread *); + struct thread *); static int uslcom_pre_param(struct ucom_softc *, struct termios *); -static void uslcom_param(struct ucom_softc *, struct termios *); -static void uslcom_get_status(struct ucom_softc *, uint8_t *, uint8_t *); static void uslcom_start_read(struct ucom_softc *); static void uslcom_stop_read(struct ucom_softc *); static void uslcom_start_write(struct ucom_softc *); @@ -182,7 +182,6 @@ static void uslcom_stop_write(struct ucom_softc *); static void uslcom_poll(struct ucom_softc *ucom); static const struct usb_config uslcom_config[USLCOM_N_TRANSFER] = { - [USLCOM_BULK_DT_WR] = { .type = UE_BULK, .endpoint = UE_ADDR_ANY, @@ -212,15 +211,15 @@ static const struct usb_config uslcom_config[USLCOM_N_TRANSFER] = { }; static struct ucom_callback uslcom_callback = { - .ucom_cfg_open = &uslcom_open, - .ucom_cfg_close = &uslcom_close, - .ucom_cfg_get_status = &uslcom_get_status, - .ucom_cfg_set_dtr = &uslcom_set_dtr, - .ucom_cfg_set_rts = &uslcom_set_rts, - .ucom_cfg_set_break = &uslcom_set_break, - .ucom_ioctl = &uslcom_ioctl, - .ucom_cfg_param = &uslcom_param, + .ucom_cfg_get_status = &uslcom_cfg_get_status, + .ucom_cfg_set_dtr = &uslcom_cfg_set_dtr, + .ucom_cfg_set_rts = &uslcom_cfg_set_rts, + .ucom_cfg_set_break = &uslcom_cfg_set_break, + .ucom_cfg_open = &uslcom_cfg_open, + .ucom_cfg_close = &uslcom_cfg_close, + .ucom_cfg_param = &uslcom_cfg_param, .ucom_pre_param = &uslcom_pre_param, + .ucom_ioctl = &uslcom_ioctl, .ucom_start_read = &uslcom_start_read, .ucom_stop_read = &uslcom_stop_read, .ucom_start_write = &uslcom_start_write, @@ -501,7 +500,7 @@ uslcom_free(struct ucom_softc *ucom) } static void -uslcom_open(struct ucom_softc *ucom) +uslcom_cfg_open(struct ucom_softc *ucom) { struct uslcom_softc *sc = ucom->sc_parent; struct usb_device_request req; @@ -512,7 +511,7 @@ uslcom_open(struct ucom_softc *ucom) USETW(req.wIndex, sc->sc_iface_no); USETW(req.wLength, 0); - if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, + if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, &req, NULL, 0, 1000)) { DPRINTF("UART enable failed (ignored)\n"); } @@ -522,7 +521,7 @@ uslcom_open(struct ucom_softc *ucom) } static void -uslcom_close(struct ucom_softc *ucom) +uslcom_cfg_close(struct ucom_softc *ucom) { struct uslcom_softc *sc = ucom->sc_parent; struct usb_device_request req; @@ -536,7 +535,7 @@ uslcom_close(struct ucom_softc *ucom) USETW(req.wIndex, sc->sc_iface_no); USETW(req.wLength, 0); - if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, + if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, &req, NULL, 0, 1000)) { DPRINTF("UART disable failed (ignored)\n"); } @@ -565,13 +564,13 @@ uslcom_get_partnum(struct uslcom_softc *sc) } static void -uslcom_set_dtr(struct ucom_softc *ucom, uint8_t onoff) +uslcom_cfg_set_dtr(struct ucom_softc *ucom, uint8_t onoff) { - struct uslcom_softc *sc = ucom->sc_parent; + struct uslcom_softc *sc = ucom->sc_parent; struct usb_device_request req; uint16_t ctl; - DPRINTF("onoff = %d\n", onoff); + DPRINTF("onoff = %d\n", onoff); ctl = onoff ? USLCOM_MHS_DTR_ON : 0; ctl |= USLCOM_MHS_DTR_SET; @@ -582,20 +581,20 @@ uslcom_set_dtr(struct ucom_softc *ucom, uint8_t onoff) USETW(req.wIndex, sc->sc_iface_no); USETW(req.wLength, 0); - if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, + if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, &req, NULL, 0, 1000)) { DPRINTF("Setting DTR failed (ignored)\n"); } } static void -uslcom_set_rts(struct ucom_softc *ucom, uint8_t onoff) +uslcom_cfg_set_rts(struct ucom_softc *ucom, uint8_t onoff) { - struct uslcom_softc *sc = ucom->sc_parent; + struct uslcom_softc *sc = ucom->sc_parent; struct usb_device_request req; uint16_t ctl; - DPRINTF("onoff = %d\n", onoff); + DPRINTF("onoff = %d\n", onoff); ctl = onoff ? USLCOM_MHS_RTS_ON : 0; ctl |= USLCOM_MHS_RTS_SET; @@ -606,7 +605,7 @@ uslcom_set_rts(struct ucom_softc *ucom, uint8_t onoff) USETW(req.wIndex, sc->sc_iface_no); USETW(req.wLength, 0); - if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, + if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, &req, NULL, 0, 1000)) { DPRINTF("Setting DTR failed (ignored)\n"); } @@ -615,13 +614,28 @@ uslcom_set_rts(struct ucom_softc *ucom, uint8_t onoff) static int uslcom_pre_param(struct ucom_softc *ucom, struct termios *t) { - if (t->c_ospeed <= 0 || t->c_ospeed > 921600) + struct uslcom_softc *sc = ucom->sc_parent; + uint32_t maxspeed; + + switch (sc->sc_partnum) { + case USLCOM_PARTNUM_CP2104: + case USLCOM_PARTNUM_CP2105: + maxspeed = 2000000; + break; + case USLCOM_PARTNUM_CP2101: + case USLCOM_PARTNUM_CP2102: + case USLCOM_PARTNUM_CP2103: + default: + maxspeed = 921600; + break; + } + if (t->c_ospeed <= 0 || t->c_ospeed > maxspeed) return (EINVAL); return (0); } static void -uslcom_param(struct ucom_softc *ucom, struct termios *t) +uslcom_cfg_param(struct ucom_softc *ucom, struct termios *t) { struct uslcom_softc *sc = ucom->sc_parent; struct usb_device_request req; @@ -637,9 +651,9 @@ uslcom_param(struct ucom_softc *ucom, struct termios *t) USETW(req.wIndex, sc->sc_iface_no); USETW(req.wLength, sizeof(baudrate)); - if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, + if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, &req, &baudrate, 0, 1000)) { - DPRINTF("Set baudrate failed (ignored)\n"); + printf("Set baudrate failed (ignored)\n"); } if (t->c_cflag & CSTOPB) @@ -674,11 +688,11 @@ uslcom_param(struct ucom_softc *ucom, struct termios *t) USETW(req.wIndex, sc->sc_iface_no); USETW(req.wLength, 0); - if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, + if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, &req, NULL, 0, 1000)) { DPRINTF("Set format failed (ignored)\n"); } - + if (t->c_cflag & CRTSCTS) { flowctrl[0] = htole32(USLCOM_FLOW_DTR_ON | USLCOM_FLOW_CTS_HS); flowctrl[1] = htole32(USLCOM_FLOW_RTS_HS); @@ -694,14 +708,14 @@ uslcom_param(struct ucom_softc *ucom, struct termios *t) USETW(req.wIndex, sc->sc_iface_no); USETW(req.wLength, sizeof(flowctrl)); - if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, + if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, &req, flowctrl, 0, 1000)) { DPRINTF("Set flowcontrol failed (ignored)\n"); } } static void -uslcom_get_status(struct ucom_softc *ucom, uint8_t *lsr, uint8_t *msr) +uslcom_cfg_get_status(struct ucom_softc *ucom, uint8_t *lsr, uint8_t *msr) { struct uslcom_softc *sc = ucom->sc_parent; @@ -713,9 +727,9 @@ uslcom_get_status(struct ucom_softc *ucom, uint8_t *lsr, uint8_t *msr) } static void -uslcom_set_break(struct ucom_softc *ucom, uint8_t onoff) +uslcom_cfg_set_break(struct ucom_softc *ucom, uint8_t onoff) { - struct uslcom_softc *sc = ucom->sc_parent; + struct uslcom_softc *sc = ucom->sc_parent; struct usb_device_request req; uint16_t brk = onoff ? USLCOM_SET_BREAK_ON : USLCOM_SET_BREAK_OFF; @@ -725,7 +739,7 @@ uslcom_set_break(struct ucom_softc *ucom, uint8_t onoff) USETW(req.wIndex, sc->sc_iface_no); USETW(req.wLength, 0); - if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, + if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, &req, NULL, 0, 1000)) { DPRINTF("Set BREAK failed (ignored)\n"); } @@ -754,7 +768,7 @@ uslcom_ioctl(struct ucom_softc *ucom, uint32_t cmd, caddr_t data, USETW(req.wIndex, sc->sc_iface_no); USETW(req.wLength, sizeof(latch)); - if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, + if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, &req, &latch, 0, 1000)) { DPRINTF("Get LATCH failed\n"); error = EIO; @@ -773,7 +787,7 @@ uslcom_ioctl(struct ucom_softc *ucom, uint32_t cmd, caddr_t data, USETW(req.wIndex, (*(int *)data)); USETW(req.wLength, 0); - if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, + if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, &req, NULL, 0, 1000)) { DPRINTF("Set LATCH failed\n"); error = EIO; @@ -888,7 +902,7 @@ uslcom_control_callback(struct usb_xfer *xfer, usb_error_t error) USETW(req.wValue, 0); USETW(req.wIndex, sc->sc_iface_no); USETW(req.wLength, sizeof(buf)); - + usbd_xfer_set_frames(xfer, 2); usbd_xfer_set_frame_len(xfer, 0, sizeof(req)); usbd_xfer_set_frame_len(xfer, 1, sizeof(buf)); diff --git a/freebsd/sys/dev/usb/usb_device.c b/freebsd/sys/dev/usb/usb_device.c index 5f1cf409..5d6b9d0f 100644 --- a/freebsd/sys/dev/usb/usb_device.c +++ b/freebsd/sys/dev/usb/usb_device.c @@ -120,7 +120,7 @@ static void usb_cdev_free(struct usb_device *); #ifdef USB_TEMPLATE int usb_template = USB_TEMPLATE; #else -int usb_template; +int usb_template = -1; #endif #ifndef __rtems__ diff --git a/freebsd/sys/dev/usb/usb_ioctl.h b/freebsd/sys/dev/usb/usb_ioctl.h index 7cf90649..fcd31e31 100644 --- a/freebsd/sys/dev/usb/usb_ioctl.h +++ b/freebsd/sys/dev/usb/usb_ioctl.h @@ -69,6 +69,7 @@ enum { USB_TEMP_PHONE, /* USB Phone */ USB_TEMP_SERIALNET, /* USB CDC Ethernet and Modem */ USB_TEMP_MIDI, /* USB MIDI */ + USB_TEMP_MULTI, /* USB Ethernet, serial, and storage */ USB_TEMP_MAX, }; diff --git a/freebsd/sys/dev/usb/usb_request.c b/freebsd/sys/dev/usb/usb_request.c index 6be241c2..cb69ce0e 100644 --- a/freebsd/sys/dev/usb/usb_request.c +++ b/freebsd/sys/dev/usb/usb_request.c @@ -1157,7 +1157,11 @@ usbd_req_get_string_any(struct usb_device *udev, struct mtx *mtx, char *buf, *s == '+' || *s == ' ' || *s == '.' || - *s == ',') { + *s == ',' || + *s == ':' || + *s == '/' || + *s == '(' || + *s == ')') { /* allowed */ s++; } diff --git a/freebsd/sys/dev/usb/wlan/if_rsu.c b/freebsd/sys/dev/usb/wlan/if_rsu.c index 179de4e5..1cb504cc 100644 --- a/freebsd/sys/dev/usb/wlan/if_rsu.c +++ b/freebsd/sys/dev/usb/wlan/if_rsu.c @@ -890,7 +890,7 @@ rsu_set_multi(struct rsu_softc *sc) TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) { ifp = vap->iv_ifp; if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { caddr_t dl; uint8_t pos; diff --git a/freebsd/sys/dev/usb/wlan/if_zyd.c b/freebsd/sys/dev/usb/wlan/if_zyd.c index dee03bdd..1835b58b 100644 --- a/freebsd/sys/dev/usb/wlan/if_zyd.c +++ b/freebsd/sys/dev/usb/wlan/if_zyd.c @@ -2002,7 +2002,7 @@ zyd_set_multi(struct zyd_softc *sc) TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) { ifp = vap->iv_ifp; if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; v = ((uint8_t *)LLADDR((struct sockaddr_dl *) diff --git a/freebsd/sys/i386/include/machine/md_var.h b/freebsd/sys/i386/include/machine/md_var.h index 38d2a828..53e1861c 100644 --- a/freebsd/sys/i386/include/machine/md_var.h +++ b/freebsd/sys/i386/include/machine/md_var.h @@ -45,14 +45,19 @@ extern int szfreebsd4_sigcode; #endif #ifdef COMPAT_43 extern int szosigcode; +extern int sz_lcall_tramp; #endif extern uint32_t *vm_page_dump; +extern vm_offset_t proc0kstack; +extern uintptr_t setidt_disp; struct segment_descriptor; union savefpu; -void bcopyb(const void *from, void *to, size_t len); +int cp_slow0(vm_offset_t uva, size_t len, bool write, + void (*f)(vm_offset_t, void *), void *arg); void cpu_switch_load_gs(void) __asm(__STRING(cpu_switch_load_gs)); +void copyout_init_tramp(void); void doreti_iret(void) __asm(__STRING(doreti_iret)); void doreti_iret_fault(void) __asm(__STRING(doreti_iret_fault)); void doreti_popl_ds(void) __asm(__STRING(doreti_popl_ds)); @@ -71,6 +76,7 @@ void ppro_reenable_apic(void); void set_fsbase(struct thread *td, uint32_t base); void set_gsbase(struct thread *td, uint32_t base); void setidt(int idx, alias_for_inthand_t *func, int typ, int dpl, int selec); +void setidt_nodisp(int idx, uintptr_t func, int typ, int dpl, int selec); union savefpu *get_pcb_user_save_td(struct thread *td); union savefpu *get_pcb_user_save_pcb(struct pcb *pcb); diff --git a/freebsd/sys/kern/init_main.c b/freebsd/sys/kern/init_main.c index 86bc11fc..42afff5e 100644 --- a/freebsd/sys/kern/init_main.c +++ b/freebsd/sys/kern/init_main.c @@ -722,7 +722,9 @@ start_init(void *dummy) vm_offset_t addr; struct execve_args args; int options, error; - char *var, *path, *next, *s; + size_t pathlen; + char *var, *path; + char *free_init_path, *tmp_init_path; char *ucp, **uap, *arg0, *arg1; struct thread *td; struct proc *p; @@ -751,17 +753,12 @@ start_init(void *dummy) strlcpy(init_path, var, sizeof(init_path)); freeenv(var); } + free_init_path = tmp_init_path = strdup(init_path, M_TEMP); - for (path = init_path; *path != '\0'; path = next) { - while (*path == ':') - path++; - if (*path == '\0') - break; - for (next = path; *next != '\0' && *next != ':'; next++) - /* nothing */ ; + while ((path = strsep(&tmp_init_path, ":")) != NULL) { + pathlen = strlen(path) + 1; if (bootverbose) - printf("start_init: trying %.*s\n", (int)(next - path), - path); + printf("start_init: trying %s\n", path); /* * Move out the boot flag argument. @@ -793,9 +790,8 @@ start_init(void *dummy) /* * Move out the file name (also arg 0). */ - (void)subyte(--ucp, 0); - for (s = next - 1; s >= path; s--) - (void)subyte(--ucp, *s); + ucp -= pathlen; + copyout(path, ucp, pathlen); arg0 = ucp; /* @@ -821,13 +817,14 @@ start_init(void *dummy) * to user mode as init! */ if ((error = sys_execve(td, &args)) == EJUSTRETURN) { + free(free_init_path, M_TEMP); TSEXIT(); return; } if (error != ENOENT) - printf("exec %.*s: error %d\n", (int)(next - path), - path, error); + printf("exec %s: error %d\n", path, error); } + free(free_init_path, M_TEMP); printf("init: not found in path %s\n", init_path); panic("no init"); } diff --git a/freebsd/sys/kern/kern_conf.c b/freebsd/sys/kern/kern_conf.c index f62e8e4d..8605cc43 100644 --- a/freebsd/sys/kern/kern_conf.c +++ b/freebsd/sys/kern/kern_conf.c @@ -902,11 +902,11 @@ make_dev(struct cdevsw *devsw, int unit, uid_t uid, gid_t gid, int mode, { struct cdev *dev; va_list ap; - int res; + int res __unused; va_start(ap, fmt); res = make_dev_credv(0, &dev, devsw, unit, NULL, uid, gid, mode, fmt, - ap); + ap); va_end(ap); KASSERT(res == 0 && dev != NULL, ("make_dev: failed make_dev_credv (error=%d)", res)); @@ -920,7 +920,7 @@ make_dev_cred(struct cdevsw *devsw, int unit, struct ucred *cr, uid_t uid, { struct cdev *dev; va_list ap; - int res; + int res __unused; va_start(ap, fmt); res = make_dev_credv(0, &dev, devsw, unit, cr, uid, gid, mode, fmt, ap); @@ -1034,7 +1034,7 @@ make_dev_alias(struct cdev *pdev, const char *fmt, ...) { struct cdev *dev; va_list ap; - int res; + int res __unused; va_start(ap, fmt); res = make_dev_alias_v(MAKEDEV_WAITOK, &dev, pdev, fmt, ap); diff --git a/freebsd/sys/kern/kern_event.c b/freebsd/sys/kern/kern_event.c index 905ef23c..33fca549 100644 --- a/freebsd/sys/kern/kern_event.c +++ b/freebsd/sys/kern/kern_event.c @@ -33,7 +33,6 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); -#include <rtems/bsd/local/opt_compat.h> #include <rtems/bsd/local/opt_ktrace.h> #include <rtems/bsd/local/opt_kqueue.h> @@ -791,7 +790,7 @@ static void filt_timerdetach(struct knote *kn) { struct kq_timer_cb_data *kc; - unsigned int old; + unsigned int old __unused; kc = kn->kn_ptr.p_v; callout_drain(&kc->c); @@ -1409,7 +1408,6 @@ kqueue_register(struct kqueue *kq, struct kevent *kev, struct thread *td, int wa struct file *fp; struct knote *kn, *tkn; struct knlist *knl; - cap_rights_t rights; int error, filt, event; int haskqglobal, filedesc_unlock; @@ -1445,8 +1443,7 @@ findkn: if (kev->ident > INT_MAX) error = EBADF; else - error = fget(td, kev->ident, - cap_rights_init(&rights, CAP_EVENT), &fp); + error = fget(td, kev->ident, &cap_event_rights, &fp); if (error) goto done; diff --git a/freebsd/sys/kern/kern_intr.c b/freebsd/sys/kern/kern_intr.c index aa896467..8f6c2a6d 100644 --- a/freebsd/sys/kern/kern_intr.c +++ b/freebsd/sys/kern/kern_intr.c @@ -63,9 +63,6 @@ __FBSDID("$FreeBSD$"); #ifndef __rtems__ #include <machine/md_var.h> #else /* __rtems__ */ - #ifdef INTR_FILTER - #error INTR_FILTER is currently not suppported with RTEMS - #endif #include <machine/rtems-bsd-thread.h> #define RTEMSBSD_SWI_WAKEUP_EVENT RTEMS_EVENT_31 #undef ticks @@ -115,26 +112,13 @@ static struct mtx event_lock; MTX_SYSINIT(intr_event_list, &event_lock, "intr event list", MTX_DEF); static void intr_event_update(struct intr_event *ie); -#ifdef INTR_FILTER -static int intr_event_schedule_thread(struct intr_event *ie, - struct intr_thread *ithd); -static int intr_filter_loop(struct intr_event *ie, - struct trapframe *frame, struct intr_thread **ithd); -static struct intr_thread *ithread_create(const char *name, - struct intr_handler *ih); -#else static int intr_event_schedule_thread(struct intr_event *ie); static struct intr_thread *ithread_create(const char *name); -#endif #ifndef __rtems__ static void ithread_destroy(struct intr_thread *ithread); #endif /* __rtems__ */ static void ithread_execute_handlers(struct proc *p, struct intr_event *ie); -#ifdef INTR_FILTER -static void priv_ithread_execute_handler(struct proc *p, - struct intr_handler *ih); -#endif static void ithread_loop(void *); static void ithread_update(struct intr_thread *ithd); #ifndef __rtems__ @@ -534,7 +518,6 @@ intr_event_destroy(struct intr_event *ie) } #endif /* __rtems__ */ -#ifndef INTR_FILTER static struct intr_thread * ithread_create(const char *name) { @@ -565,36 +548,6 @@ ithread_create(const char *name) CTR2(KTR_INTR, "%s: created %s", __func__, name); return (ithd); } -#else -#ifndef __rtems__ -static struct intr_thread * -ithread_create(const char *name, struct intr_handler *ih) -{ -#ifdef __rtems__ - struct proc *intrproc; -#endif /* __rtems__ */ - struct intr_thread *ithd; - struct thread *td; - int error; - - ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO); - - error = kproc_kthread_add(ithread_loop, ih, &intrproc, - &td, RFSTOPPED | RFHIGHPID, - 0, "intr", "%s", name); - if (error) - panic("kproc_create() failed with %d", error); - thread_lock(td); - sched_class(td, PRI_ITHD); - TD_SET_IWAIT(td); - thread_unlock(td); - td->td_pflags |= TDP_ITHREAD; - ithd->it_thread = td; - CTR2(KTR_INTR, "%s: created %s", __func__, name); - return (ithd); -} -#endif /* __rtems__ */ -#endif #ifndef __rtems__ static void @@ -614,7 +567,6 @@ ithread_destroy(struct intr_thread *ithread) } #endif /* __rtems__ */ -#ifndef INTR_FILTER int intr_event_add_handler(struct intr_event *ie, const char *name, driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, @@ -688,92 +640,6 @@ intr_event_add_handler(struct intr_event *ie, const char *name, *cookiep = ih; return (0); } -#else -#ifndef __rtems__ -int -intr_event_add_handler(struct intr_event *ie, const char *name, - driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, - enum intr_type flags, void **cookiep) -{ - struct intr_handler *ih, *temp_ih; - struct intr_thread *it; - - if (ie == NULL || name == NULL || (handler == NULL && filter == NULL)) - return (EINVAL); - - /* Allocate and populate an interrupt handler structure. */ - ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO); - ih->ih_filter = filter; - ih->ih_handler = handler; - ih->ih_argument = arg; - strlcpy(ih->ih_name, name, sizeof(ih->ih_name)); - ih->ih_event = ie; - ih->ih_pri = pri; - if (flags & INTR_EXCL) - ih->ih_flags = IH_EXCLUSIVE; - if (flags & INTR_MPSAFE) - ih->ih_flags |= IH_MPSAFE; - if (flags & INTR_ENTROPY) - ih->ih_flags |= IH_ENTROPY; - - /* We can only have one exclusive handler in a event. */ - mtx_lock(&ie->ie_lock); - if (!TAILQ_EMPTY(&ie->ie_handlers)) { - if ((flags & INTR_EXCL) || - (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) { - mtx_unlock(&ie->ie_lock); - free(ih, M_ITHREAD); - return (EINVAL); - } - } - - /* For filtered handlers, create a private ithread to run on. */ - if (filter != NULL && handler != NULL) { - mtx_unlock(&ie->ie_lock); - it = ithread_create("intr: newborn", ih); - mtx_lock(&ie->ie_lock); - it->it_event = ie; - ih->ih_thread = it; - ithread_update(it); /* XXX - do we really need this?!?!? */ - } else { /* Create the global per-event thread if we need one. */ - while (ie->ie_thread == NULL && handler != NULL) { - if (ie->ie_flags & IE_ADDING_THREAD) - msleep(ie, &ie->ie_lock, 0, "ithread", 0); - else { - ie->ie_flags |= IE_ADDING_THREAD; - mtx_unlock(&ie->ie_lock); - it = ithread_create("intr: newborn", ih); - mtx_lock(&ie->ie_lock); - ie->ie_flags &= ~IE_ADDING_THREAD; - ie->ie_thread = it; - it->it_event = ie; - ithread_update(it); - wakeup(ie); - } - } - } - - /* Add the new handler to the event in priority order. */ - TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) { - if (temp_ih->ih_pri > ih->ih_pri) - break; - } - if (temp_ih == NULL) - TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next); - else - TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next); - intr_event_update(ie); - - CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name, - ie->ie_name); - mtx_unlock(&ie->ie_lock); - - if (cookiep != NULL) - *cookiep = ih; - return (0); -} -#endif /* __rtems__ */ -#endif #ifndef __rtems__ /* @@ -892,7 +758,6 @@ _intr_drain(int irq) #endif /* __rtems__ */ -#ifndef INTR_FILTER #ifndef __rtems__ int intr_event_remove_handler(void *cookie) @@ -997,9 +862,6 @@ intr_event_schedule_thread(struct intr_event *ie) struct intr_thread *it; struct thread *td; struct thread *ctd; -#ifndef __rtems__ - struct proc *p; -#endif /* __rtems__ */ /* * If no ithread or no handlers, then we have a stray interrupt. @@ -1011,9 +873,6 @@ intr_event_schedule_thread(struct intr_event *ie) ctd = curthread; it = ie->ie_thread; td = it->it_thread; -#ifndef __rtems__ - p = td->td_proc; -#endif /* __rtems__ */ /* * If any of the handlers for this ithread claim to be good @@ -1026,7 +885,7 @@ intr_event_schedule_thread(struct intr_event *ie) } #ifndef __rtems__ - KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name)); + KASSERT(td->td_proc != NULL, ("ithread %s has no process", ie->ie_name)); #endif /* __rtems__ */ /* @@ -1042,13 +901,13 @@ intr_event_schedule_thread(struct intr_event *ie) thread_lock(td); #ifndef __rtems__ if (TD_AWAITING_INTR(td)) { - CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid, + CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, td->td_proc->p_pid, td->td_name); TD_CLR_IWAIT(td); sched_add(td, SRQ_INTR); } else { CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", - __func__, p->p_pid, td->td_name, it->it_need, td->td_state); + __func__, td->td_proc->p_pid, td->td_name, it->it_need, td->td_state); } #else /* __rtems__ */ /* Send event to wake the thread up. @@ -1061,174 +920,6 @@ intr_event_schedule_thread(struct intr_event *ie) return (0); } -#else -#ifndef __rtems__ -int -intr_event_remove_handler(void *cookie) -{ - struct intr_handler *handler = (struct intr_handler *)cookie; - struct intr_event *ie; - struct intr_thread *it; -#ifdef INVARIANTS - struct intr_handler *ih; -#endif -#ifdef notyet - int dead; -#endif - - if (handler == NULL) - return (EINVAL); - ie = handler->ih_event; - KASSERT(ie != NULL, - ("interrupt handler \"%s\" has a NULL interrupt event", - handler->ih_name)); - mtx_lock(&ie->ie_lock); - CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name, - ie->ie_name); -#ifdef INVARIANTS - TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) - if (ih == handler) - goto ok; - mtx_unlock(&ie->ie_lock); - panic("interrupt handler \"%s\" not found in interrupt event \"%s\"", - ih->ih_name, ie->ie_name); -ok: -#endif - /* - * If there are no ithreads (per event and per handler), then - * just remove the handler and return. - * XXX: Note that an INTR_FAST handler might be running on another CPU! - */ - if (ie->ie_thread == NULL && handler->ih_thread == NULL) { - TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); - mtx_unlock(&ie->ie_lock); - free(handler, M_ITHREAD); - return (0); - } - - /* Private or global ithread? */ - it = (handler->ih_thread) ? handler->ih_thread : ie->ie_thread; - /* - * If the interrupt thread is already running, then just mark this - * handler as being dead and let the ithread do the actual removal. - * - * During a cold boot while cold is set, msleep() does not sleep, - * so we have to remove the handler here rather than letting the - * thread do it. - */ - thread_lock(it->it_thread); - if (!TD_AWAITING_INTR(it->it_thread) && !cold) { - handler->ih_flags |= IH_DEAD; - - /* - * Ensure that the thread will process the handler list - * again and remove this handler if it has already passed - * it on the list. - * - * The release part of the following store ensures - * that the update of ih_flags is ordered before the - * it_need setting. See the comment before - * atomic_cmpset_acq(&ithd->it_need, ...) operation in - * the ithread_execute_handlers(). - */ - atomic_store_rel_int(&it->it_need, 1); - } else - TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); - thread_unlock(it->it_thread); - while (handler->ih_flags & IH_DEAD) - msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0); - /* - * At this point, the handler has been disconnected from the event, - * so we can kill the private ithread if any. - */ - if (handler->ih_thread) { - ithread_destroy(handler->ih_thread); - handler->ih_thread = NULL; - } - intr_event_update(ie); -#ifdef notyet - /* - * XXX: This could be bad in the case of ppbus(8). Also, I think - * this could lead to races of stale data when servicing an - * interrupt. - */ - dead = 1; - TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { - if (handler != NULL) { - dead = 0; - break; - } - } - if (dead) { - ithread_destroy(ie->ie_thread); - ie->ie_thread = NULL; - } -#endif - mtx_unlock(&ie->ie_lock); - free(handler, M_ITHREAD); - return (0); -} - -static int -intr_event_schedule_thread(struct intr_event *ie, struct intr_thread *it) -{ - struct intr_entropy entropy; - struct thread *td; - struct thread *ctd; -#ifndef __rtems__ - struct proc *p; -#endif /* __rtems__ */ - - /* - * If no ithread or no handlers, then we have a stray interrupt. - */ - if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || it == NULL) - return (EINVAL); - - ctd = curthread; - td = it->it_thread; -#ifndef __rtems__ - p = td->td_proc; -#endif /* __rtems__ */ - - /* - * If any of the handlers for this ithread claim to be good - * sources of entropy, then gather some. - */ - if (ie->ie_flags & IE_ENTROPY) { - entropy.event = (uintptr_t)ie; - entropy.td = ctd; - random_harvest_queue(&entropy, sizeof(entropy), 2, RANDOM_INTERRUPT); - } - - KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name)); - - /* - * Set it_need to tell the thread to keep running if it is already - * running. Then, lock the thread and see if we actually need to - * put it on the runqueue. - * - * Use store_rel to arrange that the store to ih_need in - * swi_sched() is before the store to it_need and prepare for - * transfer of this order to loads in the ithread. - */ - atomic_store_rel_int(&it->it_need, 1); - thread_lock(td); - if (TD_AWAITING_INTR(td)) { - CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid, - td->td_name); - TD_CLR_IWAIT(td); - sched_add(td, SRQ_INTR); - } else { - CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", - __func__, p->p_pid, td->td_name, it->it_need, td->td_state); - } - thread_unlock(td); - - return (0); -} -#endif /* __rtems__ */ -#endif /* * Allow interrupt event binding for software interrupt handlers -- a no-op, @@ -1283,7 +974,7 @@ swi_sched(void *cookie, int flags) struct intr_handler *ih = (struct intr_handler *)cookie; struct intr_event *ie = ih->ih_event; struct intr_entropy entropy; - int error; + int error __unused; CTR3(KTR_INTR, "swi_sched: %s %s need=%d", ie->ie_name, ih->ih_name, ih->ih_need); @@ -1303,11 +994,7 @@ swi_sched(void *cookie, int flags) #ifndef __rtems__ VM_CNT_INC(v_soft); #endif /* __rtems__ */ -#ifdef INTR_FILTER - error = intr_event_schedule_thread(ie, ie->ie_thread); -#else error = intr_event_schedule_thread(ie); -#endif KASSERT(error == 0, ("stray software interrupt")); } } @@ -1326,38 +1013,6 @@ swi_remove(void *cookie) return (intr_event_remove_handler(cookie)); } -#ifdef INTR_FILTER -static void -priv_ithread_execute_handler(struct proc *p, struct intr_handler *ih) -{ - struct intr_event *ie; - - ie = ih->ih_event; - /* - * If this handler is marked for death, remove it from - * the list of handlers and wake up the sleeper. - */ - if (ih->ih_flags & IH_DEAD) { - mtx_lock(&ie->ie_lock); - TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next); - ih->ih_flags &= ~IH_DEAD; - wakeup(ih); - mtx_unlock(&ie->ie_lock); - return; - } - - /* Execute this handler. */ - CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x", - __func__, p->p_pid, (void *)ih->ih_handler, ih->ih_argument, - ih->ih_name, ih->ih_flags); - - if (!(ih->ih_flags & IH_MPSAFE)) - mtx_lock(&Giant); - ih->ih_handler(ih->ih_argument); - if (!(ih->ih_flags & IH_MPSAFE)) - mtx_unlock(&Giant); -} -#endif #endif /* __rtems__ */ /* @@ -1461,7 +1116,6 @@ ithread_execute_handlers(struct proc *p, struct intr_event *ie) ie->ie_post_ithread(ie->ie_source); } -#ifndef INTR_FILTER /* * This is the main code for interrupt threads. */ @@ -1571,7 +1225,7 @@ intr_event_handle(struct intr_event *ie, struct trapframe *frame) struct intr_handler *ih; struct trapframe *oldframe; struct thread *td; - int error, ret, thread; + int ret, thread; td = curthread; @@ -1644,233 +1298,15 @@ intr_event_handle(struct intr_event *ie, struct trapframe *frame) /* Schedule the ithread if needed. */ if (thread) { - error = intr_event_schedule_thread(ie); - KASSERT(error == 0, ("bad stray interrupt")); - } - critical_exit(); - td->td_intr_nesting_level--; - return (0); -} -#endif /* __rtems__ */ -#else -#ifndef __rtems__ -/* - * This is the main code for interrupt threads. - */ -static void -ithread_loop(void *arg) -{ - struct intr_thread *ithd; - struct intr_handler *ih; - struct intr_event *ie; - struct thread *td; - struct proc *p; - int priv; - int wake; - - td = curthread; - p = td->td_proc; - ih = (struct intr_handler *)arg; - priv = (ih->ih_thread != NULL) ? 1 : 0; - ithd = (priv) ? ih->ih_thread : ih->ih_event->ie_thread; - KASSERT(ithd->it_thread == td, - ("%s: ithread and proc linkage out of sync", __func__)); - ie = ithd->it_event; - ie->ie_count = 0; - wake = 0; - - /* - * As long as we have interrupts outstanding, go through the - * list of handlers, giving each one a go at it. - */ - for (;;) { - /* - * If we are an orphaned thread, then just die. - */ - if (ithd->it_flags & IT_DEAD) { - CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__, - p->p_pid, td->td_name); - free(ithd, M_ITHREAD); - kthread_exit(); - } + int error __unused; - /* - * Service interrupts. If another interrupt arrives while - * we are running, it will set it_need to note that we - * should make another pass. - * - * The load_acq part of the following cmpset ensures - * that the load of ih_need in ithread_execute_handlers() - * is ordered after the load of it_need here. - */ - while (atomic_cmpset_acq_int(&ithd->it_need, 1, 0) != 0) { - if (priv) - priv_ithread_execute_handler(p, ih); - else - ithread_execute_handlers(p, ie); - } - WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); - mtx_assert(&Giant, MA_NOTOWNED); - - /* - * Processed all our interrupts. Now get the sched - * lock. This may take a while and it_need may get - * set again, so we have to check it again. - */ - thread_lock(td); - if (atomic_load_acq_int(&ithd->it_need) == 0 && - (ithd->it_flags & (IT_DEAD | IT_WAIT)) == 0) { - TD_SET_IWAIT(td); - ie->ie_count = 0; - mi_switch(SW_VOL | SWT_IWAIT, NULL); - } - if (ithd->it_flags & IT_WAIT) { - wake = 1; - ithd->it_flags &= ~IT_WAIT; - } - thread_unlock(td); - if (wake) { - wakeup(ithd); - wake = 0; - } - } -} - -/* - * Main loop for interrupt filter. - * - * Some architectures (i386, amd64 and arm) require the optional frame - * parameter, and use it as the main argument for fast handler execution - * when ih_argument == NULL. - * - * Return value: - * o FILTER_STRAY: No filter recognized the event, and no - * filter-less handler is registered on this - * line. - * o FILTER_HANDLED: A filter claimed the event and served it. - * o FILTER_SCHEDULE_THREAD: No filter claimed the event, but there's at - * least one filter-less handler on this line. - * o FILTER_HANDLED | - * FILTER_SCHEDULE_THREAD: A filter claimed the event, and asked for - * scheduling the per-handler ithread. - * - * In case an ithread has to be scheduled, in *ithd there will be a - * pointer to a struct intr_thread containing the thread to be - * scheduled. - */ - -static int -intr_filter_loop(struct intr_event *ie, struct trapframe *frame, - struct intr_thread **ithd) -{ - struct intr_handler *ih; - void *arg; - int ret, thread_only; - - ret = 0; - thread_only = 0; - TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { - /* - * Execute fast interrupt handlers directly. - * To support clock handlers, if a handler registers - * with a NULL argument, then we pass it a pointer to - * a trapframe as its argument. - */ - arg = ((ih->ih_argument == NULL) ? frame : ih->ih_argument); - - CTR5(KTR_INTR, "%s: exec %p/%p(%p) for %s", __func__, - ih->ih_filter, ih->ih_handler, arg, ih->ih_name); - - if (ih->ih_filter != NULL) - ret = ih->ih_filter(arg); - else { - thread_only = 1; - continue; - } - KASSERT(ret == FILTER_STRAY || - ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 && - (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0), - ("%s: incorrect return value %#x from %s", __func__, ret, - ih->ih_name)); - if (ret & FILTER_STRAY) - continue; - else { - *ithd = ih->ih_thread; - return (ret); - } - } - - /* - * No filters handled the interrupt and we have at least - * one handler without a filter. In this case, we schedule - * all of the filter-less handlers to run in the ithread. - */ - if (thread_only) { - *ithd = ie->ie_thread; - return (FILTER_SCHEDULE_THREAD); - } - return (FILTER_STRAY); -} - -/* - * Main interrupt handling body. - * - * Input: - * o ie: the event connected to this interrupt. - * o frame: some archs (i.e. i386) pass a frame to some. - * handlers as their main argument. - * Return value: - * o 0: everything ok. - * o EINVAL: stray interrupt. - */ -int -intr_event_handle(struct intr_event *ie, struct trapframe *frame) -{ - struct intr_thread *ithd; - struct trapframe *oldframe; - struct thread *td; - int thread; - - ithd = NULL; - td = curthread; - - if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers)) - return (EINVAL); - - td->td_intr_nesting_level++; - thread = 0; - critical_enter(); - oldframe = td->td_intr_frame; - td->td_intr_frame = frame; - thread = intr_filter_loop(ie, frame, &ithd); - if (thread & FILTER_HANDLED) { - if (ie->ie_post_filter != NULL) - ie->ie_post_filter(ie->ie_source); - } else { - if (ie->ie_pre_ithread != NULL) - ie->ie_pre_ithread(ie->ie_source); + error = intr_event_schedule_thread(ie); + KASSERT(error == 0, ("bad stray interrupt")); } - td->td_intr_frame = oldframe; critical_exit(); - - /* Interrupt storm logic */ - if (thread & FILTER_STRAY) { - ie->ie_count++; - if (ie->ie_count < intr_storm_threshold) - printf("Interrupt stray detection not present\n"); - } - - /* Schedule an ithread if needed. */ - if (thread & FILTER_SCHEDULE_THREAD) { - if (intr_event_schedule_thread(ie, ithd) != 0) - panic("%s: impossible stray interrupt", __func__); - } td->td_intr_nesting_level--; return (0); } -#endif /* __rtems__ */ -#endif -#ifndef __rtems__ #ifdef DDB /* diff --git a/freebsd/sys/kern/kern_linker.c b/freebsd/sys/kern/kern_linker.c index c19071a6..197ee5bb 100644 --- a/freebsd/sys/kern/kern_linker.c +++ b/freebsd/sys/kern/kern_linker.c @@ -168,7 +168,7 @@ linker_init(void *arg) TAILQ_INIT(&linker_files); } -SYSINIT(linker, SI_SUB_KLD, SI_ORDER_FIRST, linker_init, 0); +SYSINIT(linker, SI_SUB_KLD, SI_ORDER_FIRST, linker_init, NULL); static void linker_stop_class_add(void *arg) @@ -426,7 +426,7 @@ linker_init_kernel_modules(void) } SYSINIT(linker_kernel, SI_SUB_KLD, SI_ORDER_ANY, linker_init_kernel_modules, - 0); + NULL); #ifndef __rtems__ static int @@ -1700,7 +1700,7 @@ fail: /* woohoo! we made it! */ } -SYSINIT(preload, SI_SUB_KLD, SI_ORDER_MIDDLE, linker_preload, 0); +SYSINIT(preload, SI_SUB_KLD, SI_ORDER_MIDDLE, linker_preload, NULL); /* * Handle preload files that failed to load any modules. @@ -1735,7 +1735,7 @@ linker_preload_finish(void *arg) * becomes runnable in SI_SUB_KTHREAD_INIT, so go slightly before that. */ SYSINIT(preload_finish, SI_SUB_KTHREAD_INIT - 100, SI_ORDER_MIDDLE, - linker_preload_finish, 0); + linker_preload_finish, NULL); /* * Search for a not-loaded module by name. diff --git a/freebsd/sys/kern/kern_mbuf.c b/freebsd/sys/kern/kern_mbuf.c index 78e3528f..13467dfe 100644 --- a/freebsd/sys/kern/kern_mbuf.c +++ b/freebsd/sys/kern/kern_mbuf.c @@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$"); #include <sys/domain.h> #include <sys/eventhandler.h> #include <sys/kernel.h> +#include <sys/limits.h> #include <sys/lock.h> #include <sys/mutex.h> #include <sys/protosw.h> @@ -393,6 +394,199 @@ mbuf_init(void *dummy) } SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL); +#ifdef NETDUMP +/* + * netdump makes use of a pre-allocated pool of mbufs and clusters. When + * netdump is configured, we initialize a set of UMA cache zones which return + * items from this pool. At panic-time, the regular UMA zone pointers are + * overwritten with those of the cache zones so that drivers may allocate and + * free mbufs and clusters without attempting to allocate physical memory. + * + * We keep mbufs and clusters in a pair of mbuf queues. In particular, for + * the purpose of caching clusters, we treat them as mbufs. + */ +static struct mbufq nd_mbufq = + { STAILQ_HEAD_INITIALIZER(nd_mbufq.mq_head), 0, INT_MAX }; +static struct mbufq nd_clustq = + { STAILQ_HEAD_INITIALIZER(nd_clustq.mq_head), 0, INT_MAX }; + +static int nd_clsize; +static uma_zone_t nd_zone_mbuf; +static uma_zone_t nd_zone_clust; +static uma_zone_t nd_zone_pack; + +static int +nd_buf_import(void *arg, void **store, int count, int domain __unused, + int flags) +{ + struct mbufq *q; + struct mbuf *m; + int i; + + q = arg; + + for (i = 0; i < count; i++) { + m = mbufq_dequeue(q); + if (m == NULL) + break; + trash_init(m, q == &nd_mbufq ? MSIZE : nd_clsize, flags); + store[i] = m; + } + return (i); +} + +static void +nd_buf_release(void *arg, void **store, int count) +{ + struct mbufq *q; + struct mbuf *m; + int i; + + q = arg; + + for (i = 0; i < count; i++) { + m = store[i]; + (void)mbufq_enqueue(q, m); + } +} + +static int +nd_pack_import(void *arg __unused, void **store, int count, int domain __unused, + int flags __unused) +{ + struct mbuf *m; + void *clust; + int i; + + for (i = 0; i < count; i++) { + m = m_get(MT_DATA, M_NOWAIT); + if (m == NULL) + break; + clust = uma_zalloc(nd_zone_clust, M_NOWAIT); + if (clust == NULL) { + m_free(m); + break; + } + mb_ctor_clust(clust, nd_clsize, m, 0); + store[i] = m; + } + return (i); +} + +static void +nd_pack_release(void *arg __unused, void **store, int count) +{ + struct mbuf *m; + void *clust; + int i; + + for (i = 0; i < count; i++) { + m = store[i]; + clust = m->m_ext.ext_buf; + uma_zfree(nd_zone_clust, clust); + uma_zfree(nd_zone_mbuf, m); + } +} + +/* + * Free the pre-allocated mbufs and clusters reserved for netdump, and destroy + * the corresponding UMA cache zones. + */ +void +netdump_mbuf_drain(void) +{ + struct mbuf *m; + void *item; + + if (nd_zone_mbuf != NULL) { + uma_zdestroy(nd_zone_mbuf); + nd_zone_mbuf = NULL; + } + if (nd_zone_clust != NULL) { + uma_zdestroy(nd_zone_clust); + nd_zone_clust = NULL; + } + if (nd_zone_pack != NULL) { + uma_zdestroy(nd_zone_pack); + nd_zone_pack = NULL; + } + + while ((m = mbufq_dequeue(&nd_mbufq)) != NULL) + m_free(m); + while ((item = mbufq_dequeue(&nd_clustq)) != NULL) + uma_zfree(m_getzone(nd_clsize), item); +} + +/* + * Callback invoked immediately prior to starting a netdump. + */ +void +netdump_mbuf_dump(void) +{ + + /* + * All cluster zones return buffers of the size requested by the + * drivers. It's up to the driver to reinitialize the zones if the + * MTU of a netdump-enabled interface changes. + */ + printf("netdump: overwriting mbuf zone pointers\n"); + zone_mbuf = nd_zone_mbuf; + zone_clust = nd_zone_clust; + zone_pack = nd_zone_pack; + zone_jumbop = nd_zone_clust; + zone_jumbo9 = nd_zone_clust; + zone_jumbo16 = nd_zone_clust; +} + +/* + * Reinitialize the netdump mbuf+cluster pool and cache zones. + */ +void +netdump_mbuf_reinit(int nmbuf, int nclust, int clsize) +{ + struct mbuf *m; + void *item; + + netdump_mbuf_drain(); + + nd_clsize = clsize; + + nd_zone_mbuf = uma_zcache_create("netdump_" MBUF_MEM_NAME, + MSIZE, mb_ctor_mbuf, mb_dtor_mbuf, +#ifdef INVARIANTS + trash_init, trash_fini, +#else + NULL, NULL, +#endif + nd_buf_import, nd_buf_release, + &nd_mbufq, UMA_ZONE_NOBUCKET); + + nd_zone_clust = uma_zcache_create("netdump_" MBUF_CLUSTER_MEM_NAME, + clsize, mb_ctor_clust, +#ifdef INVARIANTS + trash_dtor, trash_init, trash_fini, +#else + NULL, NULL, NULL, +#endif + nd_buf_import, nd_buf_release, + &nd_clustq, UMA_ZONE_NOBUCKET); + + nd_zone_pack = uma_zcache_create("netdump_" MBUF_PACKET_MEM_NAME, + MCLBYTES, mb_ctor_pack, mb_dtor_pack, NULL, NULL, + nd_pack_import, nd_pack_release, + NULL, UMA_ZONE_NOBUCKET); + + while (nmbuf-- > 0) { + m = m_get(MT_DATA, M_WAITOK); + uma_zfree(nd_zone_mbuf, m); + } + while (nclust-- > 0) { + item = uma_zalloc(m_getzone(nd_clsize), M_WAITOK); + uma_zfree(nd_zone_clust, item); + } +} +#endif /* NETDUMP */ + /* * UMA backend page allocator for the jumbo frame zones. * @@ -702,18 +896,18 @@ mb_free_ext(struct mbuf *m) case EXT_MOD_TYPE: case EXT_DISPOSABLE: KASSERT(mref->m_ext.ext_free != NULL, - ("%s: ext_free not set", __func__)); + ("%s: ext_free not set", __func__)); mref->m_ext.ext_free(mref); uma_zfree(zone_mbuf, mref); break; case EXT_EXTREF: KASSERT(m->m_ext.ext_free != NULL, - ("%s: ext_free not set", __func__)); + ("%s: ext_free not set", __func__)); m->m_ext.ext_free(m); break; default: KASSERT(m->m_ext.ext_type == 0, - ("%s: unknown ext_type", __func__)); + ("%s: unknown ext_type", __func__)); } } diff --git a/freebsd/sys/kern/kern_mib.c b/freebsd/sys/kern/kern_mib.c index 3fd48334..cacf497d 100644 --- a/freebsd/sys/kern/kern_mib.c +++ b/freebsd/sys/kern/kern_mib.c @@ -42,7 +42,6 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); -#include <rtems/bsd/local/opt_compat.h> #include <rtems/bsd/local/opt_posix.h> #include <rtems/bsd/local/opt_config.h> @@ -100,11 +99,6 @@ SYSCTL_ROOT_NODE(OID_AUTO, regression, CTLFLAG_RW, 0, "Regression test MIB"); #endif -#ifdef EXT_RESOURCES -SYSCTL_ROOT_NODE(OID_AUTO, clock, CTLFLAG_RW, 0, - "Clocks"); -#endif - SYSCTL_STRING(_kern, OID_AUTO, ident, CTLFLAG_RD|CTLFLAG_MPSAFE, kern_ident, 0, "Kernel identifier"); diff --git a/freebsd/sys/kern/kern_module.c b/freebsd/sys/kern/kern_module.c index 58a2f83a..81686061 100644 --- a/freebsd/sys/kern/kern_module.c +++ b/freebsd/sys/kern/kern_module.c @@ -28,8 +28,6 @@ * SUCH DAMAGE. */ -#include <rtems/bsd/local/opt_compat.h> - #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); @@ -99,7 +97,7 @@ module_init(void *arg) #endif /* __rtems__ */ } -SYSINIT(module, SI_SUB_KLD, SI_ORDER_FIRST, module_init, 0); +SYSINIT(module, SI_SUB_KLD, SI_ORDER_FIRST, module_init, NULL); #ifndef __rtems__ static void diff --git a/freebsd/sys/kern/kern_synch.c b/freebsd/sys/kern/kern_synch.c index 59f00ed4..9c0d1206 100644 --- a/freebsd/sys/kern/kern_synch.c +++ b/freebsd/sys/kern/kern_synch.c @@ -117,7 +117,7 @@ sleepinit(void *unused) * vmem tries to lock the sleepq mutexes when free'ing kva, so make sure * it is available. */ -SYSINIT(sleepinit, SI_SUB_KMEM, SI_ORDER_ANY, sleepinit, 0); +SYSINIT(sleepinit, SI_SUB_KMEM, SI_ORDER_ANY, sleepinit, NULL); /* * General sleep call. Suspends the current thread until a wakeup is @@ -160,6 +160,7 @@ _sleep(void *ident, struct lock_object *lock, int priority, KASSERT(ident != NULL, ("_sleep: NULL ident")); #ifndef __rtems__ KASSERT(TD_IS_RUNNING(td), ("_sleep: curthread not running")); + KASSERT(td->td_epochnest == 0, ("sleeping in an epoch section")); if (priority & PDROP) KASSERT(lock != NULL && lock != &Giant.lock_object, ("PDROP requires a non-Giant lock")); @@ -465,8 +466,9 @@ mi_switch(int flags, struct thread *newtd) CTR4(KTR_PROC, "mi_switch: old thread %ld (td_sched %p, pid %ld, %s)", td->td_tid, td_get_sched(td), td->td_proc->p_pid, td->td_name); #ifdef KDTRACE_HOOKS - if ((flags & SW_PREEMPT) != 0 || ((flags & SW_INVOL) != 0 && - (flags & SW_TYPE_MASK) == SWT_NEEDRESCHED)) + if (__predict_false(sdt_probes_enabled) && + ((flags & SW_PREEMPT) != 0 || ((flags & SW_INVOL) != 0 && + (flags & SW_TYPE_MASK) == SWT_NEEDRESCHED))) SDT_PROBE0(sched, , , preempt); #endif sched_switch(td, newtd, flags); diff --git a/freebsd/sys/kern/kern_sysctl.c b/freebsd/sys/kern/kern_sysctl.c index 8b27a5c6..b4e9711f 100644 --- a/freebsd/sys/kern/kern_sysctl.c +++ b/freebsd/sys/kern/kern_sysctl.c @@ -43,7 +43,6 @@ __FBSDID("$FreeBSD$"); #include <rtems/bsd/local/opt_capsicum.h> -#include <rtems/bsd/local/opt_compat.h> #include <rtems/bsd/local/opt_ktrace.h> #include <sys/param.h> @@ -928,7 +927,7 @@ sysctl_register_all(void *arg) sysctl_register_oid(*oidp); SYSCTL_WUNLOCK(); } -SYSINIT(sysctl, SI_SUB_KMEM, SI_ORDER_FIRST, sysctl_register_all, 0); +SYSINIT(sysctl, SI_SUB_KMEM, SI_ORDER_FIRST, sysctl_register_all, NULL); /* * "Staff-functions" diff --git a/freebsd/sys/kern/subr_bus.c b/freebsd/sys/kern/subr_bus.c index 8076e7e3..0626ec0a 100644 --- a/freebsd/sys/kern/subr_bus.c +++ b/freebsd/sys/kern/subr_bus.c @@ -1863,6 +1863,8 @@ make_device(device_t parent, const char *name, int unit) return (NULL); } } + if (parent != NULL && device_has_quiet_children(parent)) + dev->flags |= DF_QUIET | DF_QUIET_CHILDREN; dev->ivars = NULL; dev->softc = NULL; @@ -2688,6 +2690,15 @@ device_quiet(device_t dev) } /** + * @brief Set the DF_QUIET_CHILDREN flag for the device + */ +void +device_quiet_children(device_t dev) +{ + dev->flags |= DF_QUIET_CHILDREN; +} + +/** * @brief Clear the DF_QUIET flag for the device */ void @@ -2697,6 +2708,15 @@ device_verbose(device_t dev) } /** + * @brief Return non-zero if the DF_QUIET_CHIDLREN flag is set on the device + */ +int +device_has_quiet_children(device_t dev) +{ + return ((dev->flags & DF_QUIET_CHILDREN) != 0); +} + +/** * @brief Return non-zero if the DF_QUIET flag is set on the device */ int @@ -3760,7 +3780,11 @@ bus_generic_detach(device_t dev) if (dev->state != DS_ATTACHED) return (EBUSY); - TAILQ_FOREACH(child, &dev->children, link) { + /* + * Detach children in the reverse order. + * See bus_generic_suspend for details. + */ + TAILQ_FOREACH_REVERSE(child, &dev->children, device_list, link) { if ((error = device_detach(child)) != 0) return (error); } @@ -3780,7 +3804,11 @@ bus_generic_shutdown(device_t dev) { device_t child; - TAILQ_FOREACH(child, &dev->children, link) { + /* + * Shut down children in the reverse order. + * See bus_generic_suspend for details. + */ + TAILQ_FOREACH_REVERSE(child, &dev->children, device_list, link) { device_shutdown(child); } @@ -3833,15 +3861,23 @@ int bus_generic_suspend(device_t dev) { int error; - device_t child, child2; + device_t child; - TAILQ_FOREACH(child, &dev->children, link) { + /* + * Suspend children in the reverse order. + * For most buses all children are equal, so the order does not matter. + * Other buses, such as acpi, carefully order their child devices to + * express implicit dependencies between them. For such buses it is + * safer to bring down devices in the reverse order. + */ + TAILQ_FOREACH_REVERSE(child, &dev->children, device_list, link) { error = BUS_SUSPEND_CHILD(dev, child); - if (error) { - for (child2 = TAILQ_FIRST(&dev->children); - child2 && child2 != child; - child2 = TAILQ_NEXT(child2, link)) - BUS_RESUME_CHILD(dev, child2); + if (error != 0) { + child = TAILQ_NEXT(child, link); + if (child != NULL) { + TAILQ_FOREACH_FROM(child, &dev->children, link) + BUS_RESUME_CHILD(dev, child); + } return (error); } } @@ -5285,8 +5321,9 @@ sysctl_devices(SYSCTL_HANDLER_ARGS) u_int namelen = arg2; int index; device_t dev; - struct u_device udev; /* XXX this is a bit big */ + struct u_device *udev; int error; + char *walker, *ep; if (namelen != 2) return (EINVAL); @@ -5307,24 +5344,45 @@ sysctl_devices(SYSCTL_HANDLER_ARGS) return (ENOENT); /* - * Populate the return array. + * Populate the return item, careful not to overflow the buffer. */ - bzero(&udev, sizeof(udev)); - udev.dv_handle = (uintptr_t)dev; - udev.dv_parent = (uintptr_t)dev->parent; - if (dev->nameunit != NULL) - strlcpy(udev.dv_name, dev->nameunit, sizeof(udev.dv_name)); - if (dev->desc != NULL) - strlcpy(udev.dv_desc, dev->desc, sizeof(udev.dv_desc)); - if (dev->driver != NULL && dev->driver->name != NULL) - strlcpy(udev.dv_drivername, dev->driver->name, - sizeof(udev.dv_drivername)); - bus_child_pnpinfo_str(dev, udev.dv_pnpinfo, sizeof(udev.dv_pnpinfo)); - bus_child_location_str(dev, udev.dv_location, sizeof(udev.dv_location)); - udev.dv_devflags = dev->devflags; - udev.dv_flags = dev->flags; - udev.dv_state = dev->state; - error = SYSCTL_OUT(req, &udev, sizeof(udev)); + udev = malloc(sizeof(*udev), M_BUS, M_WAITOK | M_ZERO); + if (udev == NULL) + return (ENOMEM); + udev->dv_handle = (uintptr_t)dev; + udev->dv_parent = (uintptr_t)dev->parent; + udev->dv_devflags = dev->devflags; + udev->dv_flags = dev->flags; + udev->dv_state = dev->state; + walker = udev->dv_fields; + ep = walker + sizeof(udev->dv_fields); +#define CP(src) \ + if ((src) == NULL) \ + *walker++ = '\0'; \ + else { \ + strlcpy(walker, (src), ep - walker); \ + walker += strlen(walker) + 1; \ + } \ + if (walker >= ep) \ + break; + + do { + CP(dev->nameunit); + CP(dev->desc); + CP(dev->driver != NULL ? dev->driver->name : NULL); + bus_child_pnpinfo_str(dev, walker, ep - walker); + walker += strlen(walker) + 1; + if (walker >= ep) + break; + bus_child_location_str(dev, walker, ep - walker); + walker += strlen(walker) + 1; + if (walker >= ep) + break; + *walker++ = '\0'; + } while (0); +#undef CP + error = SYSCTL_OUT(req, udev, sizeof(*udev)); + free(udev, M_BUS); return (error); } diff --git a/freebsd/sys/kern/subr_gtaskqueue.c b/freebsd/sys/kern/subr_gtaskqueue.c new file mode 100644 index 00000000..aa5c922d --- /dev/null +++ b/freebsd/sys/kern/subr_gtaskqueue.c @@ -0,0 +1,1059 @@ +#include <machine/rtems-bsd-kernel-space.h> + +/*- + * Copyright (c) 2000 Doug Rabson + * Copyright (c) 2014 Jeff Roberson + * Copyright (c) 2016 Matthew Macy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/bus.h> +#include <sys/cpuset.h> +#include <sys/interrupt.h> +#include <sys/kernel.h> +#include <sys/kthread.h> +#include <sys/libkern.h> +#include <sys/limits.h> +#include <sys/lock.h> +#include <sys/malloc.h> +#include <sys/mutex.h> +#include <sys/proc.h> +#include <sys/sched.h> +#include <sys/smp.h> +#include <sys/gtaskqueue.h> +#include <rtems/bsd/sys/unistd.h> +#include <machine/stdarg.h> +#ifdef __rtems__ +#include <machine/rtems-bsd-thread.h> +#endif /* __rtems__ */ + +static MALLOC_DEFINE(M_GTASKQUEUE, "gtaskqueue", "Group Task Queues"); +static void gtaskqueue_thread_enqueue(void *); +static void gtaskqueue_thread_loop(void *arg); + +TASKQGROUP_DEFINE(softirq, mp_ncpus, 1); +TASKQGROUP_DEFINE(config, 1, 1); + +struct gtaskqueue_busy { + struct gtask *tb_running; + TAILQ_ENTRY(gtaskqueue_busy) tb_link; +}; + +static struct gtask * const TB_DRAIN_WAITER = (struct gtask *)0x1; + +struct gtaskqueue { + STAILQ_HEAD(, gtask) tq_queue; + gtaskqueue_enqueue_fn tq_enqueue; + void *tq_context; + char *tq_name; + TAILQ_HEAD(, gtaskqueue_busy) tq_active; + struct mtx tq_mutex; + struct thread **tq_threads; + int tq_tcount; + int tq_spin; + int tq_flags; + int tq_callouts; + taskqueue_callback_fn tq_callbacks[TASKQUEUE_NUM_CALLBACKS]; + void *tq_cb_contexts[TASKQUEUE_NUM_CALLBACKS]; +}; + +#define TQ_FLAGS_ACTIVE (1 << 0) +#define TQ_FLAGS_BLOCKED (1 << 1) +#define TQ_FLAGS_UNLOCKED_ENQUEUE (1 << 2) + +#define DT_CALLOUT_ARMED (1 << 0) + +#define TQ_LOCK(tq) \ + do { \ + if ((tq)->tq_spin) \ + mtx_lock_spin(&(tq)->tq_mutex); \ + else \ + mtx_lock(&(tq)->tq_mutex); \ + } while (0) +#define TQ_ASSERT_LOCKED(tq) mtx_assert(&(tq)->tq_mutex, MA_OWNED) + +#define TQ_UNLOCK(tq) \ + do { \ + if ((tq)->tq_spin) \ + mtx_unlock_spin(&(tq)->tq_mutex); \ + else \ + mtx_unlock(&(tq)->tq_mutex); \ + } while (0) +#define TQ_ASSERT_UNLOCKED(tq) mtx_assert(&(tq)->tq_mutex, MA_NOTOWNED) + +#ifdef INVARIANTS +static void +gtask_dump(struct gtask *gtask) +{ + printf("gtask: %p ta_flags=%x ta_priority=%d ta_func=%p ta_context=%p\n", + gtask, gtask->ta_flags, gtask->ta_priority, gtask->ta_func, gtask->ta_context); +} +#endif + +static __inline int +TQ_SLEEP(struct gtaskqueue *tq, void *p, struct mtx *m, int pri, const char *wm, + int t) +{ + if (tq->tq_spin) + return (msleep_spin(p, m, wm, t)); + return (msleep(p, m, pri, wm, t)); +} + +static struct gtaskqueue * +_gtaskqueue_create(const char *name, int mflags, + taskqueue_enqueue_fn enqueue, void *context, + int mtxflags, const char *mtxname __unused) +{ + struct gtaskqueue *queue; + char *tq_name; + + tq_name = malloc(TASKQUEUE_NAMELEN, M_GTASKQUEUE, mflags | M_ZERO); + if (!tq_name) + return (NULL); + + snprintf(tq_name, TASKQUEUE_NAMELEN, "%s", (name) ? name : "taskqueue"); + + queue = malloc(sizeof(struct gtaskqueue), M_GTASKQUEUE, mflags | M_ZERO); + if (!queue) { + free(tq_name, M_GTASKQUEUE); + return (NULL); + } + + STAILQ_INIT(&queue->tq_queue); + TAILQ_INIT(&queue->tq_active); + queue->tq_enqueue = enqueue; + queue->tq_context = context; + queue->tq_name = tq_name; + queue->tq_spin = (mtxflags & MTX_SPIN) != 0; + queue->tq_flags |= TQ_FLAGS_ACTIVE; + if (enqueue == gtaskqueue_thread_enqueue) + queue->tq_flags |= TQ_FLAGS_UNLOCKED_ENQUEUE; + mtx_init(&queue->tq_mutex, tq_name, NULL, mtxflags); + + return (queue); +} + + +/* + * Signal a taskqueue thread to terminate. + */ +static void +gtaskqueue_terminate(struct thread **pp, struct gtaskqueue *tq) +{ + + while (tq->tq_tcount > 0 || tq->tq_callouts > 0) { + wakeup(tq); + TQ_SLEEP(tq, pp, &tq->tq_mutex, PWAIT, "taskqueue_destroy", 0); + } +} + +static void +gtaskqueue_free(struct gtaskqueue *queue) +{ + + TQ_LOCK(queue); + queue->tq_flags &= ~TQ_FLAGS_ACTIVE; + gtaskqueue_terminate(queue->tq_threads, queue); + KASSERT(TAILQ_EMPTY(&queue->tq_active), ("Tasks still running?")); + KASSERT(queue->tq_callouts == 0, ("Armed timeout tasks")); + mtx_destroy(&queue->tq_mutex); + free(queue->tq_threads, M_GTASKQUEUE); + free(queue->tq_name, M_GTASKQUEUE); + free(queue, M_GTASKQUEUE); +} + +int +grouptaskqueue_enqueue(struct gtaskqueue *queue, struct gtask *gtask) +{ +#ifdef INVARIANTS + if (queue == NULL) { + gtask_dump(gtask); + panic("queue == NULL"); + } +#endif + TQ_LOCK(queue); + if (gtask->ta_flags & TASK_ENQUEUED) { + TQ_UNLOCK(queue); + return (0); + } + STAILQ_INSERT_TAIL(&queue->tq_queue, gtask, ta_link); + gtask->ta_flags |= TASK_ENQUEUED; + TQ_UNLOCK(queue); + if ((queue->tq_flags & TQ_FLAGS_BLOCKED) == 0) + queue->tq_enqueue(queue->tq_context); + return (0); +} + +static void +gtaskqueue_task_nop_fn(void *context) +{ +} + +/* + * Block until all currently queued tasks in this taskqueue + * have begun execution. Tasks queued during execution of + * this function are ignored. + */ +static void +gtaskqueue_drain_tq_queue(struct gtaskqueue *queue) +{ + struct gtask t_barrier; + + if (STAILQ_EMPTY(&queue->tq_queue)) + return; + + /* + * Enqueue our barrier after all current tasks, but with + * the highest priority so that newly queued tasks cannot + * pass it. Because of the high priority, we can not use + * taskqueue_enqueue_locked directly (which drops the lock + * anyway) so just insert it at tail while we have the + * queue lock. + */ + GTASK_INIT(&t_barrier, 0, USHRT_MAX, gtaskqueue_task_nop_fn, &t_barrier); + STAILQ_INSERT_TAIL(&queue->tq_queue, &t_barrier, ta_link); + t_barrier.ta_flags |= TASK_ENQUEUED; + + /* + * Once the barrier has executed, all previously queued tasks + * have completed or are currently executing. + */ + while (t_barrier.ta_flags & TASK_ENQUEUED) + TQ_SLEEP(queue, &t_barrier, &queue->tq_mutex, PWAIT, "-", 0); +} + +/* + * Block until all currently executing tasks for this taskqueue + * complete. Tasks that begin execution during the execution + * of this function are ignored. + */ +static void +gtaskqueue_drain_tq_active(struct gtaskqueue *queue) +{ + struct gtaskqueue_busy tb_marker, *tb_first; + + if (TAILQ_EMPTY(&queue->tq_active)) + return; + + /* Block taskq_terminate().*/ + queue->tq_callouts++; + + /* + * Wait for all currently executing taskqueue threads + * to go idle. + */ + tb_marker.tb_running = TB_DRAIN_WAITER; + TAILQ_INSERT_TAIL(&queue->tq_active, &tb_marker, tb_link); + while (TAILQ_FIRST(&queue->tq_active) != &tb_marker) + TQ_SLEEP(queue, &tb_marker, &queue->tq_mutex, PWAIT, "-", 0); + TAILQ_REMOVE(&queue->tq_active, &tb_marker, tb_link); + + /* + * Wakeup any other drain waiter that happened to queue up + * without any intervening active thread. + */ + tb_first = TAILQ_FIRST(&queue->tq_active); + if (tb_first != NULL && tb_first->tb_running == TB_DRAIN_WAITER) + wakeup(tb_first); + + /* Release taskqueue_terminate(). */ + queue->tq_callouts--; + if ((queue->tq_flags & TQ_FLAGS_ACTIVE) == 0) + wakeup_one(queue->tq_threads); +} + +void +gtaskqueue_block(struct gtaskqueue *queue) +{ + + TQ_LOCK(queue); + queue->tq_flags |= TQ_FLAGS_BLOCKED; + TQ_UNLOCK(queue); +} + +void +gtaskqueue_unblock(struct gtaskqueue *queue) +{ + + TQ_LOCK(queue); + queue->tq_flags &= ~TQ_FLAGS_BLOCKED; + if (!STAILQ_EMPTY(&queue->tq_queue)) + queue->tq_enqueue(queue->tq_context); + TQ_UNLOCK(queue); +} + +static void +gtaskqueue_run_locked(struct gtaskqueue *queue) +{ + struct gtaskqueue_busy tb; + struct gtaskqueue_busy *tb_first; + struct gtask *gtask; + + KASSERT(queue != NULL, ("tq is NULL")); + TQ_ASSERT_LOCKED(queue); + tb.tb_running = NULL; + + while (STAILQ_FIRST(&queue->tq_queue)) { + TAILQ_INSERT_TAIL(&queue->tq_active, &tb, tb_link); + + /* + * Carefully remove the first task from the queue and + * clear its TASK_ENQUEUED flag + */ + gtask = STAILQ_FIRST(&queue->tq_queue); + KASSERT(gtask != NULL, ("task is NULL")); + STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link); + gtask->ta_flags &= ~TASK_ENQUEUED; + tb.tb_running = gtask; + TQ_UNLOCK(queue); + + KASSERT(gtask->ta_func != NULL, ("task->ta_func is NULL")); + gtask->ta_func(gtask->ta_context); + + TQ_LOCK(queue); + tb.tb_running = NULL; + wakeup(gtask); + + TAILQ_REMOVE(&queue->tq_active, &tb, tb_link); + tb_first = TAILQ_FIRST(&queue->tq_active); + if (tb_first != NULL && + tb_first->tb_running == TB_DRAIN_WAITER) + wakeup(tb_first); + } +} + +static int +task_is_running(struct gtaskqueue *queue, struct gtask *gtask) +{ + struct gtaskqueue_busy *tb; + + TQ_ASSERT_LOCKED(queue); + TAILQ_FOREACH(tb, &queue->tq_active, tb_link) { + if (tb->tb_running == gtask) + return (1); + } + return (0); +} + +static int +gtaskqueue_cancel_locked(struct gtaskqueue *queue, struct gtask *gtask) +{ + + if (gtask->ta_flags & TASK_ENQUEUED) + STAILQ_REMOVE(&queue->tq_queue, gtask, gtask, ta_link); + gtask->ta_flags &= ~TASK_ENQUEUED; + return (task_is_running(queue, gtask) ? EBUSY : 0); +} + +int +gtaskqueue_cancel(struct gtaskqueue *queue, struct gtask *gtask) +{ + int error; + + TQ_LOCK(queue); + error = gtaskqueue_cancel_locked(queue, gtask); + TQ_UNLOCK(queue); + + return (error); +} + +void +gtaskqueue_drain(struct gtaskqueue *queue, struct gtask *gtask) +{ + + if (!queue->tq_spin) + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, __func__); + + TQ_LOCK(queue); + while ((gtask->ta_flags & TASK_ENQUEUED) || task_is_running(queue, gtask)) + TQ_SLEEP(queue, gtask, &queue->tq_mutex, PWAIT, "-", 0); + TQ_UNLOCK(queue); +} + +void +gtaskqueue_drain_all(struct gtaskqueue *queue) +{ + + if (!queue->tq_spin) + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, __func__); + + TQ_LOCK(queue); + gtaskqueue_drain_tq_queue(queue); + gtaskqueue_drain_tq_active(queue); + TQ_UNLOCK(queue); +} + +static int +_gtaskqueue_start_threads(struct gtaskqueue **tqp, int count, int pri, + cpuset_t *mask, const char *name, va_list ap) +{ + char ktname[MAXCOMLEN + 1]; + struct thread *td; + struct gtaskqueue *tq; + int i, error; + + if (count <= 0) + return (EINVAL); + + vsnprintf(ktname, sizeof(ktname), name, ap); + tq = *tqp; + + tq->tq_threads = malloc(sizeof(struct thread *) * count, M_GTASKQUEUE, + M_NOWAIT | M_ZERO); + if (tq->tq_threads == NULL) { + printf("%s: no memory for %s threads\n", __func__, ktname); + return (ENOMEM); + } + + for (i = 0; i < count; i++) { + if (count == 1) + error = kthread_add(gtaskqueue_thread_loop, tqp, NULL, + &tq->tq_threads[i], RFSTOPPED, 0, "%s", ktname); + else + error = kthread_add(gtaskqueue_thread_loop, tqp, NULL, + &tq->tq_threads[i], RFSTOPPED, 0, + "%s_%d", ktname, i); + if (error) { + /* should be ok to continue, taskqueue_free will dtrt */ + printf("%s: kthread_add(%s): error %d", __func__, + ktname, error); + tq->tq_threads[i] = NULL; /* paranoid */ + } else + tq->tq_tcount++; + } + for (i = 0; i < count; i++) { + if (tq->tq_threads[i] == NULL) + continue; + td = tq->tq_threads[i]; + if (mask) { +#ifndef __rtems__ + error = cpuset_setthread(td->td_tid, mask); + /* + * Failing to pin is rarely an actual fatal error; + * it'll just affect performance. + */ + if (error) + printf("%s: curthread=%llu: can't pin; " + "error=%d\n", + __func__, + (unsigned long long) td->td_tid, + error); +#else /* __rtems__ */ + rtems_status_code sc; + + sc = rtems_task_set_affinity(rtems_bsd_get_task_id(td), + sizeof(*mask), mask); + if (sc != RTEMS_SUCCESSFUL) + printf("%s: cannot set affinity\n", __func__); +#endif /* __rtems__ */ + } +#ifndef __rtems__ + thread_lock(td); + sched_prio(td, pri); + sched_add(td, SRQ_BORING); + thread_unlock(td); +#endif /* __rtems__ */ + } + + return (0); +} + +static int +gtaskqueue_start_threads(struct gtaskqueue **tqp, int count, int pri, + const char *name, ...) +{ + va_list ap; + int error; + + va_start(ap, name); + error = _gtaskqueue_start_threads(tqp, count, pri, NULL, name, ap); + va_end(ap); + return (error); +} + +static inline void +gtaskqueue_run_callback(struct gtaskqueue *tq, + enum taskqueue_callback_type cb_type) +{ + taskqueue_callback_fn tq_callback; + + TQ_ASSERT_UNLOCKED(tq); + tq_callback = tq->tq_callbacks[cb_type]; + if (tq_callback != NULL) + tq_callback(tq->tq_cb_contexts[cb_type]); +} + +static void +gtaskqueue_thread_loop(void *arg) +{ + struct gtaskqueue **tqp, *tq; + + tqp = arg; + tq = *tqp; + gtaskqueue_run_callback(tq, TASKQUEUE_CALLBACK_TYPE_INIT); + TQ_LOCK(tq); + while ((tq->tq_flags & TQ_FLAGS_ACTIVE) != 0) { + /* XXX ? */ + gtaskqueue_run_locked(tq); + /* + * Because taskqueue_run() can drop tq_mutex, we need to + * check if the TQ_FLAGS_ACTIVE flag wasn't removed in the + * meantime, which means we missed a wakeup. + */ + if ((tq->tq_flags & TQ_FLAGS_ACTIVE) == 0) + break; + TQ_SLEEP(tq, tq, &tq->tq_mutex, 0, "-", 0); + } + gtaskqueue_run_locked(tq); + /* + * This thread is on its way out, so just drop the lock temporarily + * in order to call the shutdown callback. This allows the callback + * to look at the taskqueue, even just before it dies. + */ + TQ_UNLOCK(tq); + gtaskqueue_run_callback(tq, TASKQUEUE_CALLBACK_TYPE_SHUTDOWN); + TQ_LOCK(tq); + + /* rendezvous with thread that asked us to terminate */ + tq->tq_tcount--; + wakeup_one(tq->tq_threads); + TQ_UNLOCK(tq); + kthread_exit(); +} + +static void +gtaskqueue_thread_enqueue(void *context) +{ + struct gtaskqueue **tqp, *tq; + + tqp = context; + tq = *tqp; + wakeup_one(tq); +} + + +static struct gtaskqueue * +gtaskqueue_create_fast(const char *name, int mflags, + taskqueue_enqueue_fn enqueue, void *context) +{ + return _gtaskqueue_create(name, mflags, enqueue, context, + MTX_SPIN, "fast_taskqueue"); +} + + +struct taskqgroup_cpu { + LIST_HEAD(, grouptask) tgc_tasks; + struct gtaskqueue *tgc_taskq; + int tgc_cnt; + int tgc_cpu; +}; + +struct taskqgroup { + struct taskqgroup_cpu tqg_queue[MAXCPU]; + struct mtx tqg_lock; + const char * tqg_name; + int tqg_adjusting; + int tqg_stride; + int tqg_cnt; +}; + +struct taskq_bind_task { + struct gtask bt_task; + int bt_cpuid; +}; + +static void +taskqgroup_cpu_create(struct taskqgroup *qgroup, int idx, int cpu) +{ + struct taskqgroup_cpu *qcpu; + + qcpu = &qgroup->tqg_queue[idx]; + LIST_INIT(&qcpu->tgc_tasks); + qcpu->tgc_taskq = gtaskqueue_create_fast(NULL, M_WAITOK, + taskqueue_thread_enqueue, &qcpu->tgc_taskq); + gtaskqueue_start_threads(&qcpu->tgc_taskq, 1, PI_SOFT, + "%s_%d", qgroup->tqg_name, idx); + qcpu->tgc_cpu = cpu; +} + +static void +taskqgroup_cpu_remove(struct taskqgroup *qgroup, int idx) +{ + + gtaskqueue_free(qgroup->tqg_queue[idx].tgc_taskq); +} + +/* + * Find the taskq with least # of tasks that doesn't currently have any + * other queues from the uniq identifier. + */ +static int +taskqgroup_find(struct taskqgroup *qgroup, void *uniq) +{ + struct grouptask *n; + int i, idx, mincnt; + int strict; + + mtx_assert(&qgroup->tqg_lock, MA_OWNED); + if (qgroup->tqg_cnt == 0) + return (0); + idx = -1; + mincnt = INT_MAX; + /* + * Two passes; First scan for a queue with the least tasks that + * does not already service this uniq id. If that fails simply find + * the queue with the least total tasks; + */ + for (strict = 1; mincnt == INT_MAX; strict = 0) { + for (i = 0; i < qgroup->tqg_cnt; i++) { + if (qgroup->tqg_queue[i].tgc_cnt > mincnt) + continue; + if (strict) { + LIST_FOREACH(n, + &qgroup->tqg_queue[i].tgc_tasks, gt_list) + if (n->gt_uniq == uniq) + break; + if (n != NULL) + continue; + } + mincnt = qgroup->tqg_queue[i].tgc_cnt; + idx = i; + } + } + if (idx == -1) + panic("taskqgroup_find: Failed to pick a qid."); + + return (idx); +} + +#ifndef __rtems__ +/* + * smp_started is unusable since it is not set for UP kernels or even for + * SMP kernels when there is 1 CPU. This is usually handled by adding a + * (mp_ncpus == 1) test, but that would be broken here since we need to + * to synchronize with the SI_SUB_SMP ordering. Even in the pure SMP case + * smp_started only gives a fuzzy ordering relative to SI_SUB_SMP. + * + * So maintain our own flag. It must be set after all CPUs are started + * and before SI_SUB_SMP:SI_ORDER_ANY so that the SYSINIT for delayed + * adjustment is properly delayed. SI_ORDER_FOURTH is clearly before + * SI_ORDER_ANY and unclearly after the CPUs are started. It would be + * simpler for adjustment to pass a flag indicating if it is delayed. + */ + +static int tqg_smp_started; + +static void +tqg_record_smp_started(void *arg) +{ + tqg_smp_started = 1; +} + +SYSINIT(tqg_record_smp_started, SI_SUB_SMP, SI_ORDER_FOURTH, + tqg_record_smp_started, NULL); +#else /* __rtems__ */ +#define tqg_smp_started 1 +#endif /* __rtems__ */ + +void +taskqgroup_attach(struct taskqgroup *qgroup, struct grouptask *gtask, + void *uniq, int irq, const char *name) +{ +#ifndef __rtems__ + cpuset_t mask; + int qid, error; +#else /* __rtems__ */ + int qid; +#endif /* __rtems__ */ + + gtask->gt_uniq = uniq; + snprintf(gtask->gt_name, GROUPTASK_NAMELEN, "%s", name ? name : "grouptask"); + gtask->gt_irq = irq; + gtask->gt_cpu = -1; + mtx_lock(&qgroup->tqg_lock); + qid = taskqgroup_find(qgroup, uniq); + qgroup->tqg_queue[qid].tgc_cnt++; + LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); + gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; +#ifndef __rtems__ + if (irq != -1 && tqg_smp_started) { + gtask->gt_cpu = qgroup->tqg_queue[qid].tgc_cpu; + CPU_ZERO(&mask); + CPU_SET(qgroup->tqg_queue[qid].tgc_cpu, &mask); + mtx_unlock(&qgroup->tqg_lock); + error = intr_setaffinity(irq, CPU_WHICH_IRQ, &mask); + if (error) + printf("%s: setaffinity failed for %s: %d\n", __func__, gtask->gt_name, error); + } else +#else /* __rtems__ */ + BSD_ASSERT(irq == -1); +#endif /* __rtems__ */ + mtx_unlock(&qgroup->tqg_lock); +} + +static void +taskqgroup_attach_deferred(struct taskqgroup *qgroup, struct grouptask *gtask) +{ +#ifndef __rtems__ + cpuset_t mask; + int qid, cpu, error; +#else /* __rtems__ */ + int qid; +#endif /* __rtems__ */ + + mtx_lock(&qgroup->tqg_lock); + qid = taskqgroup_find(qgroup, gtask->gt_uniq); +#ifndef __rtems__ + cpu = qgroup->tqg_queue[qid].tgc_cpu; + if (gtask->gt_irq != -1) { + mtx_unlock(&qgroup->tqg_lock); + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); + error = intr_setaffinity(gtask->gt_irq, CPU_WHICH_IRQ, &mask); + mtx_lock(&qgroup->tqg_lock); + if (error) + printf("%s: %s setaffinity failed: %d\n", __func__, gtask->gt_name, error); + + } +#else /* __rtems__ */ + BSD_ASSERT(gtask->gt_irq == -1); +#endif /* __rtems__ */ + qgroup->tqg_queue[qid].tgc_cnt++; + + LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, + gt_list); + MPASS(qgroup->tqg_queue[qid].tgc_taskq != NULL); + gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; + mtx_unlock(&qgroup->tqg_lock); +} + +int +taskqgroup_attach_cpu(struct taskqgroup *qgroup, struct grouptask *gtask, + void *uniq, int cpu, int irq, const char *name) +{ +#ifndef __rtems__ + cpuset_t mask; + int i, qid, error; +#else /* __rtems__ */ + int i, qid; +#endif /* __rtems__ */ + + qid = -1; + gtask->gt_uniq = uniq; + snprintf(gtask->gt_name, GROUPTASK_NAMELEN, "%s", name ? name : "grouptask"); + gtask->gt_irq = irq; + gtask->gt_cpu = cpu; + mtx_lock(&qgroup->tqg_lock); + if (tqg_smp_started) { + for (i = 0; i < qgroup->tqg_cnt; i++) + if (qgroup->tqg_queue[i].tgc_cpu == cpu) { + qid = i; + break; + } + if (qid == -1) { + mtx_unlock(&qgroup->tqg_lock); + printf("%s: qid not found for %s cpu=%d\n", __func__, gtask->gt_name, cpu); + return (EINVAL); + } + } else + qid = 0; + qgroup->tqg_queue[qid].tgc_cnt++; + LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); + gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; +#ifndef __rtems__ + cpu = qgroup->tqg_queue[qid].tgc_cpu; +#endif /* __rtems__ */ + mtx_unlock(&qgroup->tqg_lock); + +#ifndef __rtems__ + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); + if (irq != -1 && tqg_smp_started) { + error = intr_setaffinity(irq, CPU_WHICH_IRQ, &mask); + if (error) + printf("%s: setaffinity failed: %d\n", __func__, error); + } +#else /* __rtems__ */ + BSD_ASSERT(irq == -1); +#endif /* __rtems__ */ + return (0); +} + +static int +taskqgroup_attach_cpu_deferred(struct taskqgroup *qgroup, struct grouptask *gtask) +{ +#ifndef __rtems__ + cpuset_t mask; + int i, qid, irq, cpu, error; +#else /* __rtems__ */ + int i, qid, irq, cpu; +#endif /* __rtems__ */ + + qid = -1; + irq = gtask->gt_irq; + cpu = gtask->gt_cpu; + MPASS(tqg_smp_started); + mtx_lock(&qgroup->tqg_lock); + for (i = 0; i < qgroup->tqg_cnt; i++) + if (qgroup->tqg_queue[i].tgc_cpu == cpu) { + qid = i; + break; + } + if (qid == -1) { + mtx_unlock(&qgroup->tqg_lock); + printf("%s: qid not found for %s cpu=%d\n", __func__, gtask->gt_name, cpu); + return (EINVAL); + } + qgroup->tqg_queue[qid].tgc_cnt++; + LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); + MPASS(qgroup->tqg_queue[qid].tgc_taskq != NULL); + gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; + mtx_unlock(&qgroup->tqg_lock); + +#ifndef __rtems__ + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); + + if (irq != -1) { + error = intr_setaffinity(irq, CPU_WHICH_IRQ, &mask); + if (error) + printf("%s: setaffinity failed: %d\n", __func__, error); + } +#else /* __rtems__ */ + BSD_ASSERT(irq == -1); +#endif /* __rtems__ */ + return (0); +} + +void +taskqgroup_detach(struct taskqgroup *qgroup, struct grouptask *gtask) +{ + int i; + + mtx_lock(&qgroup->tqg_lock); + for (i = 0; i < qgroup->tqg_cnt; i++) + if (qgroup->tqg_queue[i].tgc_taskq == gtask->gt_taskqueue) + break; + if (i == qgroup->tqg_cnt) + panic("taskqgroup_detach: task %s not in group\n", gtask->gt_name); + qgroup->tqg_queue[i].tgc_cnt--; + LIST_REMOVE(gtask, gt_list); + mtx_unlock(&qgroup->tqg_lock); + gtask->gt_taskqueue = NULL; +} + +static void +taskqgroup_binder(void *ctx) +{ + struct taskq_bind_task *gtask = (struct taskq_bind_task *)ctx; + cpuset_t mask; +#ifndef __rtems__ + int error; +#else /* __rtems__ */ + rtems_status_code sc; +#endif /* __rtems__ */ + + CPU_ZERO(&mask); + CPU_SET(gtask->bt_cpuid, &mask); +#ifndef __rtems__ + error = cpuset_setthread(curthread->td_tid, &mask); + thread_lock(curthread); + sched_bind(curthread, gtask->bt_cpuid); + thread_unlock(curthread); + + if (error) + printf("%s: setaffinity failed: %d\n", __func__, + error); +#else /* __rtems__ */ + sc = rtems_task_set_affinity(RTEMS_SELF, sizeof(mask), &mask); + if (sc != RTEMS_SUCCESSFUL) + printf("%s: cannot set affinity\n", __func__); +#endif /* __rtems__ */ + free(gtask, M_DEVBUF); +} + +static void +taskqgroup_bind(struct taskqgroup *qgroup) +{ + struct taskq_bind_task *gtask; + int i; + + /* + * Bind taskqueue threads to specific CPUs, if they have been assigned + * one. + */ + if (qgroup->tqg_cnt == 1) + return; + + for (i = 0; i < qgroup->tqg_cnt; i++) { + gtask = malloc(sizeof (*gtask), M_DEVBUF, M_WAITOK); + GTASK_INIT(>ask->bt_task, 0, 0, taskqgroup_binder, gtask); + gtask->bt_cpuid = qgroup->tqg_queue[i].tgc_cpu; + grouptaskqueue_enqueue(qgroup->tqg_queue[i].tgc_taskq, + >ask->bt_task); + } +} + +static int +_taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride) +{ + LIST_HEAD(, grouptask) gtask_head = LIST_HEAD_INITIALIZER(NULL); + struct grouptask *gtask; + int i, k, old_cnt, old_cpu, cpu; + + mtx_assert(&qgroup->tqg_lock, MA_OWNED); + + if (cnt < 1 || cnt * stride > mp_ncpus || !tqg_smp_started) { + printf("%s: failed cnt: %d stride: %d " + "mp_ncpus: %d tqg_smp_started: %d\n", + __func__, cnt, stride, mp_ncpus, tqg_smp_started); + return (EINVAL); + } + if (qgroup->tqg_adjusting) { + printf("%s failed: adjusting\n", __func__); + return (EBUSY); + } + qgroup->tqg_adjusting = 1; + old_cnt = qgroup->tqg_cnt; + old_cpu = 0; + if (old_cnt < cnt) + old_cpu = qgroup->tqg_queue[old_cnt].tgc_cpu; + mtx_unlock(&qgroup->tqg_lock); + /* + * Set up queue for tasks added before boot. + */ + if (old_cnt == 0) { + LIST_SWAP(>ask_head, &qgroup->tqg_queue[0].tgc_tasks, + grouptask, gt_list); + qgroup->tqg_queue[0].tgc_cnt = 0; + } + + /* + * If new taskq threads have been added. + */ + cpu = old_cpu; + for (i = old_cnt; i < cnt; i++) { + taskqgroup_cpu_create(qgroup, i, cpu); + + for (k = 0; k < stride; k++) + cpu = CPU_NEXT(cpu); + } + mtx_lock(&qgroup->tqg_lock); + qgroup->tqg_cnt = cnt; + qgroup->tqg_stride = stride; + + /* + * Adjust drivers to use new taskqs. + */ + for (i = 0; i < old_cnt; i++) { + while ((gtask = LIST_FIRST(&qgroup->tqg_queue[i].tgc_tasks))) { + LIST_REMOVE(gtask, gt_list); + qgroup->tqg_queue[i].tgc_cnt--; + LIST_INSERT_HEAD(>ask_head, gtask, gt_list); + } + } + mtx_unlock(&qgroup->tqg_lock); + + while ((gtask = LIST_FIRST(>ask_head))) { + LIST_REMOVE(gtask, gt_list); + if (gtask->gt_cpu == -1) + taskqgroup_attach_deferred(qgroup, gtask); + else if (taskqgroup_attach_cpu_deferred(qgroup, gtask)) + taskqgroup_attach_deferred(qgroup, gtask); + } + +#ifdef INVARIANTS + mtx_lock(&qgroup->tqg_lock); + for (i = 0; i < qgroup->tqg_cnt; i++) { + MPASS(qgroup->tqg_queue[i].tgc_taskq != NULL); + LIST_FOREACH(gtask, &qgroup->tqg_queue[i].tgc_tasks, gt_list) + MPASS(gtask->gt_taskqueue != NULL); + } + mtx_unlock(&qgroup->tqg_lock); +#endif + /* + * If taskq thread count has been reduced. + */ + for (i = cnt; i < old_cnt; i++) + taskqgroup_cpu_remove(qgroup, i); + + taskqgroup_bind(qgroup); + + mtx_lock(&qgroup->tqg_lock); + qgroup->tqg_adjusting = 0; + + return (0); +} + +int +taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride) +{ + int error; + + mtx_lock(&qgroup->tqg_lock); + error = _taskqgroup_adjust(qgroup, cnt, stride); + mtx_unlock(&qgroup->tqg_lock); + + return (error); +} + +struct taskqgroup * +taskqgroup_create(const char *name) +{ + struct taskqgroup *qgroup; + + qgroup = malloc(sizeof(*qgroup), M_GTASKQUEUE, M_WAITOK | M_ZERO); + mtx_init(&qgroup->tqg_lock, "taskqgroup", NULL, MTX_DEF); + qgroup->tqg_name = name; + LIST_INIT(&qgroup->tqg_queue[0].tgc_tasks); + + return (qgroup); +} + +void +taskqgroup_destroy(struct taskqgroup *qgroup) +{ + +} + +void +taskqgroup_config_gtask_init(void *ctx, struct grouptask *gtask, gtask_fn_t *fn, + const char *name) +{ + + GROUPTASK_INIT(gtask, 0, fn, ctx); + taskqgroup_attach(qgroup_config, gtask, gtask, -1, name); +} + +void +taskqgroup_config_gtask_deinit(struct grouptask *gtask) +{ + taskqgroup_detach(qgroup_config, gtask); +} diff --git a/freebsd/sys/kern/subr_lock.c b/freebsd/sys/kern/subr_lock.c index 4f577b65..c2587cd0 100644 --- a/freebsd/sys/kern/subr_lock.c +++ b/freebsd/sys/kern/subr_lock.c @@ -171,8 +171,10 @@ void lock_delay_default_init(struct lock_delay_config *lc) { - lc->base = lock_roundup_2(mp_ncpus) / 4; - lc->max = lc->base * 1024; + lc->base = 1; + lc->max = lock_roundup_2(mp_ncpus) * 256; + if (lc->max > 32678) + lc->max = 32678; } #endif /* __rtems__ */ diff --git a/freebsd/sys/kern/subr_pcpu.c b/freebsd/sys/kern/subr_pcpu.c index 67a1f528..1b866e3a 100644 --- a/freebsd/sys/kern/subr_pcpu.c +++ b/freebsd/sys/kern/subr_pcpu.c @@ -130,7 +130,7 @@ dpcpu_startup(void *dummy __unused) TAILQ_INSERT_HEAD(&dpcpu_head, df, df_link); sx_init(&dpcpu_lock, "dpcpu alloc lock"); } -SYSINIT(dpcpu, SI_SUB_KLD, SI_ORDER_FIRST, dpcpu_startup, 0); +SYSINIT(dpcpu, SI_SUB_KLD, SI_ORDER_FIRST, dpcpu_startup, NULL); #endif /* __rtems__ */ /* diff --git a/freebsd/sys/kern/subr_prf.c b/freebsd/sys/kern/subr_prf.c index 12a0825d..4c45bcfe 100644 --- a/freebsd/sys/kern/subr_prf.c +++ b/freebsd/sys/kern/subr_prf.c @@ -687,6 +687,7 @@ kvprintf(char const *fmt, void (*func)(int, void*), void *arg, int radix, va_lis int stop = 0, retval = 0; num = 0; + q = NULL; if (!func) d = (char *) arg; else diff --git a/freebsd/sys/kern/subr_sleepqueue.c b/freebsd/sys/kern/subr_sleepqueue.c index b9de1580..65bd8dcc 100644 --- a/freebsd/sys/kern/subr_sleepqueue.c +++ b/freebsd/sys/kern/subr_sleepqueue.c @@ -433,7 +433,7 @@ sleepq_set_timeout_sbt(void *wchan, sbintime_t sbt, sbintime_t pr, int flags) { #ifndef __rtems__ - struct sleepqueue_chain *sc; + struct sleepqueue_chain *sc __unused; struct thread *td; sbintime_t pr1; @@ -982,7 +982,7 @@ sleepq_type(void *wchan) static int sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri) { - struct sleepqueue_chain *sc; + struct sleepqueue_chain *sc __unused; #ifdef __rtems__ Thread_Control *thread; ISR_lock_Context lock_context; @@ -1022,7 +1022,6 @@ sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri) td->td_sleepqueue = LIST_FIRST(&sq->sq_free); LIST_REMOVE(td->td_sleepqueue, sq_hash); #ifdef __rtems__ - (void)sc; thread = td->td_thread; _ISR_lock_ISR_disable(&lock_context); _Thread_Wait_acquire_default_critical(thread, &lock_context); @@ -1228,7 +1227,7 @@ sleepq_remove_matching(struct sleepqueue *sq, int queue, static void sleepq_timeout(void *arg) { - struct sleepqueue_chain *sc; + struct sleepqueue_chain *sc __unused; struct sleepqueue *sq; struct thread *td; void *wchan; diff --git a/freebsd/sys/kern/subr_uio.c b/freebsd/sys/kern/subr_uio.c index 58db0ffc..c14aea8d 100644 --- a/freebsd/sys/kern/subr_uio.c +++ b/freebsd/sys/kern/subr_uio.c @@ -222,9 +222,9 @@ uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault) int error; #endif /* __rtems__ */ - error = 0; - #ifndef __rtems__ + save = error = 0; + KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE, ("uiomove: mode")); KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread, @@ -290,7 +290,7 @@ uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault) } out: #ifndef __rtems__ - if (uio->uio_segflg == UIO_USERSPACE) + if (save) curthread_pflags_restore(save); #endif /* __rtems__ */ return (error); diff --git a/freebsd/sys/kern/sys_generic.c b/freebsd/sys/kern/sys_generic.c index 0d5193c7..9e41f0f6 100644 --- a/freebsd/sys/kern/sys_generic.c +++ b/freebsd/sys/kern/sys_generic.c @@ -42,7 +42,6 @@ __FBSDID("$FreeBSD$"); #include <rtems/bsd/local/opt_capsicum.h> -#include <rtems/bsd/local/opt_compat.h> #include <rtems/bsd/local/opt_ktrace.h> #include <sys/param.h> @@ -201,9 +200,7 @@ struct read_args { }; #endif int -sys_read(td, uap) - struct thread *td; - struct read_args *uap; +sys_read(struct thread *td, struct read_args *uap) { struct uio auio; struct iovec aiov; @@ -296,10 +293,9 @@ int kern_readv(struct thread *td, int fd, struct uio *auio) { struct file *fp; - cap_rights_t rights; int error; - error = fget_read(td, fd, cap_rights_init(&rights, CAP_READ), &fp); + error = fget_read(td, fd, &cap_read_rights, &fp); if (error) return (error); error = dofileread(td, fd, fp, auio, (off_t)-1, 0); @@ -333,17 +329,12 @@ sys_preadv(struct thread *td, struct preadv_args *uap) } int -kern_preadv(td, fd, auio, offset) - struct thread *td; - int fd; - struct uio *auio; - off_t offset; +kern_preadv(struct thread *td, int fd, struct uio *auio, off_t offset) { struct file *fp; - cap_rights_t rights; int error; - error = fget_read(td, fd, cap_rights_init(&rights, CAP_PREAD), &fp); + error = fget_read(td, fd, &cap_pread_rights, &fp); if (error) return (error); if (!(fp->f_ops->fo_flags & DFLAG_SEEKABLE)) @@ -362,13 +353,8 @@ kern_preadv(td, fd, auio, offset) * from a file using the passed in uio, offset, and flags. */ static int -dofileread(td, fd, fp, auio, offset, flags) - struct thread *td; - int fd; - struct file *fp; - struct uio *auio; - off_t offset; - int flags; +dofileread(struct thread *td, int fd, struct file *fp, struct uio *auio, + off_t offset, int flags) { ssize_t cnt; int error; @@ -415,9 +401,7 @@ struct write_args { }; #endif int -sys_write(td, uap) - struct thread *td; - struct write_args *uap; +sys_write(struct thread *td, struct write_args *uap) { struct uio auio; struct iovec aiov; @@ -511,10 +495,9 @@ int kern_writev(struct thread *td, int fd, struct uio *auio) { struct file *fp; - cap_rights_t rights; int error; - error = fget_write(td, fd, cap_rights_init(&rights, CAP_WRITE), &fp); + error = fget_write(td, fd, &cap_write_rights, &fp); if (error) return (error); error = dofilewrite(td, fd, fp, auio, (off_t)-1, 0); @@ -548,17 +531,12 @@ sys_pwritev(struct thread *td, struct pwritev_args *uap) } int -kern_pwritev(td, fd, auio, offset) - struct thread *td; - struct uio *auio; - int fd; - off_t offset; +kern_pwritev(struct thread *td, int fd, struct uio *auio, off_t offset) { struct file *fp; - cap_rights_t rights; int error; - error = fget_write(td, fd, cap_rights_init(&rights, CAP_PWRITE), &fp); + error = fget_write(td, fd, &cap_pwrite_rights, &fp); if (error) return (error); if (!(fp->f_ops->fo_flags & DFLAG_SEEKABLE)) @@ -577,13 +555,8 @@ kern_pwritev(td, fd, auio, offset) * a file using the passed in uio, offset, and flags. */ static int -dofilewrite(td, fd, fp, auio, offset, flags) - struct thread *td; - int fd; - struct file *fp; - struct uio *auio; - off_t offset; - int flags; +dofilewrite(struct thread *td, int fd, struct file *fp, struct uio *auio, + off_t offset, int flags) { ssize_t cnt; int error; @@ -632,19 +605,15 @@ dofilewrite(td, fd, fp, auio, offset, flags) * descriptor isn't writable. */ int -kern_ftruncate(td, fd, length) - struct thread *td; - int fd; - off_t length; +kern_ftruncate(struct thread *td, int fd, off_t length) { struct file *fp; - cap_rights_t rights; int error; AUDIT_ARG_FD(fd); if (length < 0) return (EINVAL); - error = fget(td, fd, cap_rights_init(&rights, CAP_FTRUNCATE), &fp); + error = fget(td, fd, &cap_ftruncate_rights, &fp); if (error) return (error); AUDIT_ARG_FILE(td->td_proc, fp); @@ -665,9 +634,7 @@ struct ftruncate_args { }; #endif int -sys_ftruncate(td, uap) - struct thread *td; - struct ftruncate_args *uap; +sys_ftruncate(struct thread *td, struct ftruncate_args *uap) { return (kern_ftruncate(td, uap->fd, uap->length)); @@ -681,9 +648,7 @@ struct oftruncate_args { }; #endif int -oftruncate(td, uap) - struct thread *td; - struct oftruncate_args *uap; +oftruncate(struct thread *td, struct oftruncate_args *uap) { return (kern_ftruncate(td, uap->fd, uap->length)); @@ -772,9 +737,6 @@ kern_ioctl(struct thread *td, int fd, u_long com, caddr_t data) { struct file *fp; struct filedesc *fdp; -#ifndef CAPABILITIES - cap_rights_t rights; -#endif int error, tmp, locked; AUDIT_ARG_FD(fd); @@ -813,7 +775,7 @@ kern_ioctl(struct thread *td, int fd, u_long com, caddr_t data) locked = LA_UNLOCKED; } #else - error = fget(td, fd, cap_rights_init(&rights, CAP_IOCTL), &fp); + error = fget(td, fd, &cap_ioctl_rights, &fp); if (error != 0) { fp = NULL; goto out; @@ -1284,11 +1246,8 @@ selsetbits(fd_mask **ibits, fd_mask **obits, int idx, fd_mask bit, int events) static __inline int getselfd_cap(struct filedesc *fdp, int fd, struct file **fpp) { - cap_rights_t rights; - cap_rights_init(&rights, CAP_EVENT); - - return (fget_unlocked(fdp, fd, &rights, fpp, NULL)); + return (fget_unlocked(fdp, fd, &cap_event_rights, fpp, NULL)); } /* @@ -1342,10 +1301,7 @@ selrescan(struct thread *td, fd_mask **ibits, fd_mask **obits) * each selinfo. */ static int -selscan(td, ibits, obits, nfd) - struct thread *td; - fd_mask **ibits, **obits; - int nfd; +selscan(struct thread *td, fd_mask **ibits, fd_mask **obits, int nfd) { struct filedesc *fdp; struct file *fp; @@ -1573,9 +1529,6 @@ pollrescan(struct thread *td) struct filedesc *fdp; struct file *fp; struct pollfd *fd; -#ifdef CAPABILITIES - cap_rights_t rights; -#endif int n; n = 0; @@ -1600,8 +1553,7 @@ pollrescan(struct thread *td) #endif /* __rtems__ */ #ifdef CAPABILITIES if (fp == NULL || - cap_check(cap_rights(fdp, fd->fd), - cap_rights_init(&rights, CAP_EVENT)) != 0) + cap_check(cap_rights(fdp, fd->fd), &cap_event_rights) != 0) #else if (fp == NULL) #endif @@ -1630,11 +1582,7 @@ pollrescan(struct thread *td) static int -pollout(td, fds, ufds, nfd) - struct thread *td; - struct pollfd *fds; - struct pollfd *ufds; - u_int nfd; +pollout(struct thread *td, struct pollfd *fds, struct pollfd *ufds, u_int nfd) { int error = 0; u_int i = 0; @@ -1655,10 +1603,7 @@ pollout(td, fds, ufds, nfd) } static int -pollscan(td, fds, nfd) - struct thread *td; - struct pollfd *fds; - u_int nfd; +pollscan(struct thread *td, struct pollfd *fds, u_int nfd) { #ifndef __rtems__ struct filedesc *fdp = td->td_proc->p_fd; @@ -1666,9 +1611,6 @@ pollscan(td, fds, nfd) struct filedesc *fdp = NULL; #endif /* __rtems__ */ struct file *fp; -#ifdef CAPABILITIES - cap_rights_t rights; -#endif int i, n = 0; FILEDESC_SLOCK(fdp); @@ -1690,8 +1632,7 @@ pollscan(td, fds, nfd) #endif /* __rtems__ */ #ifdef CAPABILITIES if (fp == NULL || - cap_check(cap_rights(fdp, fds->fd), - cap_rights_init(&rights, CAP_EVENT)) != 0) + cap_check(cap_rights(fdp, fds->fd), &cap_event_rights) != 0) #else if (fp == NULL) #endif @@ -1822,8 +1763,7 @@ selfdfree(struct seltd *stp, struct selfd *sfp) /* Drain the waiters tied to all the selfd belonging the specified selinfo. */ void -seldrain(sip) - struct selinfo *sip; +seldrain(struct selinfo *sip) { /* @@ -1841,9 +1781,7 @@ seldrain(sip) * Record a select request. */ void -selrecord(selector, sip) - struct thread *selector; - struct selinfo *sip; +selrecord(struct thread *selector, struct selinfo *sip) { struct selfd *sfp; struct seltd *stp; @@ -1892,17 +1830,14 @@ selrecord(selector, sip) /* Wake up a selecting thread. */ void -selwakeup(sip) - struct selinfo *sip; +selwakeup(struct selinfo *sip) { doselwakeup(sip, -1); } /* Wake up a selecting thread, and set its priority. */ void -selwakeuppri(sip, pri) - struct selinfo *sip; - int pri; +selwakeuppri(struct selinfo *sip, int pri) { doselwakeup(sip, pri); } @@ -1911,9 +1846,7 @@ selwakeuppri(sip, pri) * Do a wakeup when a selectable event occurs. */ static void -doselwakeup(sip, pri) - struct selinfo *sip; - int pri; +doselwakeup(struct selinfo *sip, int pri) { struct selfd *sfp; struct selfd *sfn; diff --git a/freebsd/sys/kern/sys_pipe.c b/freebsd/sys/kern/sys_pipe.c index cc5b123c..e527495a 100755 --- a/freebsd/sys/kern/sys_pipe.c +++ b/freebsd/sys/kern/sys_pipe.c @@ -93,8 +93,6 @@ * in the structure may have changed. */ -#include <rtems/bsd/local/opt_compat.h> - #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); diff --git a/freebsd/sys/kern/tty.c b/freebsd/sys/kern/tty.c index 1350c1d9..f4a2b01f 100644 --- a/freebsd/sys/kern/tty.c +++ b/freebsd/sys/kern/tty.c @@ -35,7 +35,6 @@ __FBSDID("$FreeBSD$"); #include <rtems/bsd/local/opt_capsicum.h> -#include <rtems/bsd/local/opt_compat.h> #include <sys/param.h> #include <sys/capsicum.h> diff --git a/freebsd/sys/kern/tty_inq.c b/freebsd/sys/kern/tty_inq.c index b470cb5a..8d557a55 100644 --- a/freebsd/sys/kern/tty_inq.c +++ b/freebsd/sys/kern/tty_inq.c @@ -330,7 +330,7 @@ ttyinq_write(struct ttyinq *ti, const void *buf, size_t nbytes, int quote) int ttyinq_write_nofrag(struct ttyinq *ti, const void *buf, size_t nbytes, int quote) { - size_t ret; + size_t ret __unused; if (ttyinq_bytesleft(ti) < nbytes) return (-1); diff --git a/freebsd/sys/kern/tty_outq.c b/freebsd/sys/kern/tty_outq.c index 121e9975..1643fe40 100644 --- a/freebsd/sys/kern/tty_outq.c +++ b/freebsd/sys/kern/tty_outq.c @@ -326,7 +326,7 @@ ttyoutq_write(struct ttyoutq *to, const void *buf, size_t nbytes) int ttyoutq_write_nofrag(struct ttyoutq *to, const void *buf, size_t nbytes) { - size_t ret; + size_t ret __unused; if (ttyoutq_bytesleft(to) < nbytes) return (-1); diff --git a/freebsd/sys/kern/uipc_mbuf.c b/freebsd/sys/kern/uipc_mbuf.c index 7bf531e0..185d14a0 100644 --- a/freebsd/sys/kern/uipc_mbuf.c +++ b/freebsd/sys/kern/uipc_mbuf.c @@ -1635,9 +1635,6 @@ m_unshare(struct mbuf *m0, int how) mprev->m_len += m->m_len; mprev->m_next = m->m_next; /* unlink from chain */ m_free(m); /* reclaim mbuf */ -#if 0 - newipsecstat.ips_mbcoalesced++; -#endif } else { mprev = m; } @@ -1667,9 +1664,6 @@ m_unshare(struct mbuf *m0, int how) mprev->m_len += m->m_len; mprev->m_next = m->m_next; /* unlink from chain */ m_free(m); /* reclaim mbuf */ -#if 0 - newipsecstat.ips_clcoalesced++; -#endif continue; } diff --git a/freebsd/sys/kern/uipc_sockbuf.c b/freebsd/sys/kern/uipc_sockbuf.c index 0c4ace6b..ec493c04 100644 --- a/freebsd/sys/kern/uipc_sockbuf.c +++ b/freebsd/sys/kern/uipc_sockbuf.c @@ -466,6 +466,7 @@ sbsetopt(struct socket *so, int cmd, u_long cc) u_int *hiwat, *lowat; int error; + sb = NULL; SOCK_LOCK(so); if (SOLISTENING(so)) { switch (cmd) { diff --git a/freebsd/sys/kern/uipc_socket.c b/freebsd/sys/kern/uipc_socket.c index 43763026..e82642e4 100644 --- a/freebsd/sys/kern/uipc_socket.c +++ b/freebsd/sys/kern/uipc_socket.c @@ -109,7 +109,6 @@ __FBSDID("$FreeBSD$"); #include <rtems/bsd/local/opt_inet.h> #include <rtems/bsd/local/opt_inet6.h> -#include <rtems/bsd/local/opt_compat.h> #include <rtems/bsd/local/opt_sctp.h> #include <sys/param.h> diff --git a/freebsd/sys/kern/uipc_syscalls.c b/freebsd/sys/kern/uipc_syscalls.c index 6f3c95c0..0872aa62 100644 --- a/freebsd/sys/kern/uipc_syscalls.c +++ b/freebsd/sys/kern/uipc_syscalls.c @@ -39,7 +39,6 @@ __FBSDID("$FreeBSD$"); #include <rtems/bsd/local/opt_capsicum.h> #include <rtems/bsd/local/opt_inet.h> #include <rtems/bsd/local/opt_inet6.h> -#include <rtems/bsd/local/opt_compat.h> #include <rtems/bsd/local/opt_ktrace.h> #include <sys/param.h> @@ -300,12 +299,16 @@ kern_bindat(struct thread *td, int dirfd, int fd, struct sockaddr *sa) { struct socket *so; struct file *fp; - cap_rights_t rights; int error; +#ifdef CAPABILITY_MODE + if (IN_CAPABILITY_MODE(td) && (dirfd == AT_FDCWD)) + return (ECAPMODE); +#endif + AUDIT_ARG_FD(fd); AUDIT_ARG_SOCKADDR(td, dirfd, sa); - error = getsock_cap(td, fd, cap_rights_init(&rights, CAP_BIND), + error = getsock_cap(td, fd, &cap_bind_rights, &fp, NULL, NULL); if (error != 0) return (error); @@ -364,11 +367,10 @@ kern_listen(struct thread *td, int s, int backlog) { struct socket *so; struct file *fp; - cap_rights_t rights; int error; AUDIT_ARG_FD(s); - error = getsock_cap(td, s, cap_rights_init(&rights, CAP_LISTEN), + error = getsock_cap(td, s, &cap_listen_rights, &fp, NULL, NULL); if (error == 0) { so = fp->f_data; @@ -493,7 +495,6 @@ kern_accept4(struct thread *td, int s, struct sockaddr **name, struct sockaddr *sa = NULL; struct socket *head, *so; struct filecaps fcaps; - cap_rights_t rights; u_int fflag; pid_t pgid; int error, fd, tmp; @@ -502,7 +503,7 @@ kern_accept4(struct thread *td, int s, struct sockaddr **name, *name = NULL; AUDIT_ARG_FD(s); - error = getsock_cap(td, s, cap_rights_init(&rights, CAP_ACCEPT), + error = getsock_cap(td, s, &cap_accept_rights, &headfp, &fflag, &fcaps); if (error != 0) return (error); @@ -692,12 +693,16 @@ kern_connectat(struct thread *td, int dirfd, int fd, struct sockaddr *sa) { struct socket *so; struct file *fp; - cap_rights_t rights; int error, interrupted = 0; +#ifdef CAPABILITY_MODE + if (IN_CAPABILITY_MODE(td) && (dirfd == AT_FDCWD)) + return (ECAPMODE); +#endif + AUDIT_ARG_FD(fd); AUDIT_ARG_SOCKADDR(td, dirfd, sa); - error = getsock_cap(td, fd, cap_rights_init(&rights, CAP_CONNECT), + error = getsock_cap(td, fd, &cap_connect_rights, &fp, NULL, NULL); if (error != 0) return (error); @@ -986,7 +991,7 @@ kern_sendit(struct thread *td, int s, struct msghdr *mp, int flags, struct iovec *iov; struct socket *so; #ifndef __rtems__ - cap_rights_t rights; + cap_rights_t *rights; #endif /* __rtems__ */ #ifdef KTRACE struct uio *ktruio = NULL; @@ -996,13 +1001,14 @@ kern_sendit(struct thread *td, int s, struct msghdr *mp, int flags, AUDIT_ARG_FD(s); #ifndef __rtems__ + rights = &cap_send_rights; cap_rights_init(&rights, CAP_SEND); if (mp->msg_name != NULL) { AUDIT_ARG_SOCKADDR(td, AT_FDCWD, mp->msg_name); - cap_rights_set(&rights, CAP_CONNECT); + rights = &cap_send_connect_rights; } #endif /* __rtems__ */ - error = getsock_cap(td, s, &rights, &fp, NULL, NULL); + error = getsock_cap(td, s, rights, &fp, NULL, NULL); if (error != 0) { m_freem(control); return (error); @@ -1259,7 +1265,6 @@ kern_recvit(struct thread *td, int s, struct msghdr *mp, enum uio_seg fromseg, struct file *fp; struct socket *so; struct sockaddr *fromsa = NULL; - cap_rights_t rights; #ifdef KTRACE struct uio *ktruio = NULL; #endif @@ -1270,7 +1275,7 @@ kern_recvit(struct thread *td, int s, struct msghdr *mp, enum uio_seg fromseg, *controlp = NULL; AUDIT_ARG_FD(s); - error = getsock_cap(td, s, cap_rights_init(&rights, CAP_RECV), + error = getsock_cap(td, s, &cap_recv_rights, &fp, NULL, NULL); if (error != 0) return (error); @@ -1613,11 +1618,10 @@ kern_shutdown(struct thread *td, int s, int how) { struct socket *so; struct file *fp; - cap_rights_t rights; int error; AUDIT_ARG_FD(s); - error = getsock_cap(td, s, cap_rights_init(&rights, CAP_SHUTDOWN), + error = getsock_cap(td, s, &cap_shutdown_rights, &fp, NULL, NULL); if (error == 0) { so = fp->f_data; @@ -1696,7 +1700,6 @@ kern_setsockopt(struct thread *td, int s, int level, int name, void *val, struct socket *so; struct file *fp; struct sockopt sopt; - cap_rights_t rights; int error; if (val == NULL && valsize != 0) @@ -1721,7 +1724,7 @@ kern_setsockopt(struct thread *td, int s, int level, int name, void *val, } AUDIT_ARG_FD(s); - error = getsock_cap(td, s, cap_rights_init(&rights, CAP_SETSOCKOPT), + error = getsock_cap(td, s, &cap_setsockopt_rights, &fp, NULL, NULL); if (error == 0) { so = fp->f_data; @@ -1792,7 +1795,6 @@ kern_getsockopt(struct thread *td, int s, int level, int name, void *val, struct socket *so; struct file *fp; struct sockopt sopt; - cap_rights_t rights; int error; if (val == NULL) @@ -1817,7 +1819,7 @@ kern_getsockopt(struct thread *td, int s, int level, int name, void *val, } AUDIT_ARG_FD(s); - error = getsock_cap(td, s, cap_rights_init(&rights, CAP_GETSOCKOPT), + error = getsock_cap(td, s, &cap_getsockopt_rights, &fp, NULL, NULL); if (error == 0) { so = fp->f_data; @@ -1892,12 +1894,11 @@ kern_getsockname(struct thread *td, int fd, struct sockaddr **sa, { struct socket *so; struct file *fp; - cap_rights_t rights; socklen_t len; int error; AUDIT_ARG_FD(fd); - error = getsock_cap(td, fd, cap_rights_init(&rights, CAP_GETSOCKNAME), + error = getsock_cap(td, fd, &cap_getsockname_rights, &fp, NULL, NULL); if (error != 0) return (error); @@ -2008,12 +2009,11 @@ kern_getpeername(struct thread *td, int fd, struct sockaddr **sa, { struct socket *so; struct file *fp; - cap_rights_t rights; socklen_t len; int error; AUDIT_ARG_FD(fd); - error = getsock_cap(td, fd, cap_rights_init(&rights, CAP_GETPEERNAME), + error = getsock_cap(td, fd, &cap_getpeername_rights, &fp, NULL, NULL); if (error != 0) return (error); diff --git a/freebsd/sys/kern/uipc_usrreq.c b/freebsd/sys/kern/uipc_usrreq.c index 7849be9d..688682d4 100644 --- a/freebsd/sys/kern/uipc_usrreq.c +++ b/freebsd/sys/kern/uipc_usrreq.c @@ -4,9 +4,9 @@ * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 1982, 1986, 1989, 1991, 1993 - * The Regents of the University of California. - * Copyright (c) 2004-2009 Robert N. M. Watson - * All rights reserved. + * The Regents of the University of California. All Rights Reserved. + * Copyright (c) 2004-2009 Robert N. M. Watson All Rights Reserved. + * Copyright (c) 2018 Matthew Macy * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -204,12 +204,40 @@ SYSCTL_INT(_net_local, OID_AUTO, deferred, CTLFLAG_RD, /* * Locking and synchronization: * - * Two types of locks exist in the local domain socket implementation: a - * a global linkage rwlock and per-unpcb mutexes. The linkage lock protects - * the socket count, global generation number, stream/datagram global lists and - * interconnection of unpcbs, the v_socket and unp_vnode pointers, and can be - * held exclusively over the acquisition of multiple unpcb locks to prevent - * deadlock. + * Three types of locks exist in the local domain socket implementation: a + * a global linkage rwlock, the mtxpool lock, and per-unpcb mutexes. + * The linkage lock protects the socket count, global generation number, + * and stream/datagram global lists. + * + * The mtxpool lock protects the vnode from being modified while referenced. + * Lock ordering requires that it be acquired before any unpcb locks. + * + * The unpcb lock (unp_mtx) protects all fields in the unpcb. Of particular + * note is that this includes the unp_conn field. So long as the unpcb lock + * is held the reference to the unpcb pointed to by unp_conn is valid. If we + * require that the unpcb pointed to by unp_conn remain live in cases where + * we need to drop the unp_mtx as when we need to acquire the lock for a + * second unpcb the caller must first acquire an additional reference on the + * second unpcb and then revalidate any state (typically check that unp_conn + * is non-NULL) upon requiring the initial unpcb lock. The lock ordering + * between unpcbs is the conventional ascending address order. Two helper + * routines exist for this: + * + * - unp_pcb_lock2(unp, unp2) - which just acquires the two locks in the + * safe ordering. + * + * - unp_pcb_owned_lock2(unp, unp2, freed) - the lock for unp is held + * when called. If unp is unlocked and unp2 is subsequently freed + * freed will be set to 1. + * + * The helper routines for references are: + * + * - unp_pcb_hold(unp): Can be called any time we currently hold a valid + * reference to unp. + * + * - unp_pcb_rele(unp): The caller must hold the unp lock. If we are + * releasing the last reference, detach must have been called thus + * unp->unp_socket be NULL. * * UNIX domain sockets each have an unpcb hung off of their so_pcb pointer, * allocated in pru_attach() and freed in pru_detach(). The validity of that @@ -223,15 +251,8 @@ SYSCTL_INT(_net_local, OID_AUTO, deferred, CTLFLAG_RD, * to the unpcb is held. Typically, this reference will be from the socket, * or from another unpcb when the referring unpcb's lock is held (in order * that the reference not be invalidated during use). For example, to follow - * unp->unp_conn->unp_socket, you need unlock the lock on unp, not unp_conn, - * as unp_socket remains valid as long as the reference to unp_conn is valid. - * - * Fields of unpcbss are locked using a per-unpcb lock, unp_mtx. Individual - * atomic reads without the lock may be performed "lockless", but more - * complex reads and read-modify-writes require the mutex to be held. No - * lock order is defined between unpcb locks -- multiple unpcb locks may be - * acquired at the same time only when holding the linkage rwlock - * exclusively, which prevents deadlocks. + * unp->unp_conn->unp_socket, you need to hold a lock on unp_conn to guarantee + * that detach is not run clearing unp_socket. * * Blocking with UNIX domain sockets is a tricky issue: unlike most network * protocols, bind() is a non-atomic operation, and connect() requires @@ -270,13 +291,19 @@ static struct mtx unp_defers_lock; #define UNP_DEFERRED_LOCK() mtx_lock(&unp_defers_lock) #define UNP_DEFERRED_UNLOCK() mtx_unlock(&unp_defers_lock) +#define UNP_REF_LIST_LOCK() UNP_DEFERRED_LOCK(); +#define UNP_REF_LIST_UNLOCK() UNP_DEFERRED_UNLOCK(); + #define UNP_PCB_LOCK_INIT(unp) mtx_init(&(unp)->unp_mtx, \ - "unp_mtx", "unp_mtx", \ - MTX_DUPOK|MTX_DEF|MTX_RECURSE) + "unp", "unp", \ + MTX_DUPOK|MTX_DEF) #define UNP_PCB_LOCK_DESTROY(unp) mtx_destroy(&(unp)->unp_mtx) #define UNP_PCB_LOCK(unp) mtx_lock(&(unp)->unp_mtx) +#define UNP_PCB_TRYLOCK(unp) mtx_trylock(&(unp)->unp_mtx) #define UNP_PCB_UNLOCK(unp) mtx_unlock(&(unp)->unp_mtx) +#define UNP_PCB_OWNED(unp) mtx_owned(&(unp)->unp_mtx) #define UNP_PCB_LOCK_ASSERT(unp) mtx_assert(&(unp)->unp_mtx, MA_OWNED) +#define UNP_PCB_UNLOCK_ASSERT(unp) mtx_assert(&(unp)->unp_mtx, MA_NOTOWNED) static int uipc_connect2(struct socket *, struct socket *); static int uipc_ctloutput(struct socket *, struct sockopt *); @@ -308,6 +335,77 @@ static struct mbuf *unp_addsockcred(struct thread *, struct mbuf *); static void unp_process_defers(void * __unused, int); #endif /* __rtems__ */ + +static void +unp_pcb_hold(struct unpcb *unp) +{ + MPASS(unp->unp_refcount); + refcount_acquire(&unp->unp_refcount); +} + +static int +unp_pcb_rele(struct unpcb *unp) +{ + int freed; + + UNP_PCB_LOCK_ASSERT(unp); + MPASS(unp->unp_refcount); + if ((freed = refcount_release(&unp->unp_refcount))) { + /* we got here with having detached? */ + MPASS(unp->unp_socket == NULL); + UNP_PCB_UNLOCK(unp); + UNP_PCB_LOCK_DESTROY(unp); + uma_zfree(unp_zone, unp); + } + return (freed); +} + +static void +unp_pcb_lock2(struct unpcb *unp, struct unpcb *unp2) +{ + MPASS(unp != unp2); + UNP_PCB_UNLOCK_ASSERT(unp); + UNP_PCB_UNLOCK_ASSERT(unp2); + if ((uintptr_t)unp2 > (uintptr_t)unp) { + UNP_PCB_LOCK(unp); + UNP_PCB_LOCK(unp2); + } else { + UNP_PCB_LOCK(unp2); + UNP_PCB_LOCK(unp); + } +} + +static __noinline void +unp_pcb_owned_lock2_slowpath(struct unpcb *unp, struct unpcb **unp2p, int *freed) + +{ + struct unpcb *unp2; + + unp2 = *unp2p; + unp_pcb_hold((unp2)); + UNP_PCB_UNLOCK((unp)); + UNP_PCB_LOCK((unp2)); + UNP_PCB_LOCK((unp)); + *freed = unp_pcb_rele((unp2)); + if (*freed) + *unp2p = NULL; +} + +#define unp_pcb_owned_lock2(unp, unp2, freed) do { \ + freed = 0; \ + UNP_PCB_LOCK_ASSERT((unp)); \ + UNP_PCB_UNLOCK_ASSERT((unp2)); \ + MPASS(unp != unp2); \ + if (__predict_true(UNP_PCB_TRYLOCK((unp2)))) \ + break; \ + else if ((uintptr_t)(unp2) > (uintptr_t)(unp)) \ + UNP_PCB_LOCK((unp2)); \ + else { \ + unp_pcb_owned_lock2_slowpath((unp), &(unp2), &freed); \ + } \ +} while (0) + + /* * Definitions of protocols supported in the LOCAL domain. */ @@ -365,17 +463,16 @@ uipc_abort(struct socket *so) unp = sotounpcb(so); KASSERT(unp != NULL, ("uipc_abort: unp == NULL")); + UNP_PCB_UNLOCK_ASSERT(unp); - UNP_LINK_WLOCK(); UNP_PCB_LOCK(unp); unp2 = unp->unp_conn; if (unp2 != NULL) { - UNP_PCB_LOCK(unp2); + unp_pcb_hold(unp2); + UNP_PCB_UNLOCK(unp); unp_drop(unp2); - UNP_PCB_UNLOCK(unp2); - } - UNP_PCB_UNLOCK(unp); - UNP_LINK_WUNLOCK(); + } else + UNP_PCB_UNLOCK(unp); } static int @@ -636,7 +733,6 @@ restart: #endif /* __rtems__ */ soun = (struct sockaddr_un *)sodupsockaddr(nam, M_WAITOK); - UNP_LINK_WLOCK(); UNP_PCB_LOCK(unp); #ifndef __rtems__ VOP_UNP_BIND(vp, unp); @@ -645,7 +741,6 @@ restart: unp->unp_addr = soun; unp->unp_flags &= ~UNP_BINDING; UNP_PCB_UNLOCK(unp); - UNP_LINK_WUNLOCK(); #ifndef __rtems__ VOP_UNLOCK(vp, 0); vn_finished_write(mp); @@ -676,9 +771,7 @@ uipc_connect(struct socket *so, struct sockaddr *nam, struct thread *td) int error; KASSERT(td == curthread, ("uipc_connect: td != curthread")); - UNP_LINK_WLOCK(); error = unp_connect(so, nam, td); - UNP_LINK_WUNLOCK(); return (error); } @@ -689,9 +782,7 @@ uipc_connectat(int fd, struct socket *so, struct sockaddr *nam, int error; KASSERT(td == curthread, ("uipc_connectat: td != curthread")); - UNP_LINK_WLOCK(); error = unp_connectat(fd, so, nam, td); - UNP_LINK_WUNLOCK(); return (error); } @@ -701,30 +792,53 @@ uipc_close(struct socket *so) struct unpcb *unp, *unp2; #ifndef __rtems__ struct vnode *vp = NULL; + struct mtx *vplock; #else /* __rtems__ */ IMFS_generic_t *vp = NULL; #endif /* __rtems__ */ - + int freed; unp = sotounpcb(so); KASSERT(unp != NULL, ("uipc_close: unp == NULL")); - UNP_LINK_WLOCK(); + +#ifndef __rtems__ + vplock = NULL; +#endif /* __rtems__ */ + if ((vp = unp->unp_vnode) != NULL) { +#ifndef __rtems__ + vplock = mtx_pool_find(mtxpool_sleep, vp); + mtx_lock(vplock); +#endif /* __rtems__ */ + } UNP_PCB_LOCK(unp); - unp2 = unp->unp_conn; - if (unp2 != NULL) { - UNP_PCB_LOCK(unp2); - unp_disconnect(unp, unp2); - UNP_PCB_UNLOCK(unp2); + if (vp && unp->unp_vnode == NULL) { +#ifndef __rtems__ + mtx_unlock(vplock); +#endif /* __rtems__ */ + vp = NULL; } - if (SOLISTENING(so) && ((vp = unp->unp_vnode) != NULL)) { + if (vp != NULL) { VOP_UNP_DETACH(vp); unp->unp_vnode = NULL; } - UNP_PCB_UNLOCK(unp); - UNP_LINK_WUNLOCK(); + unp2 = unp->unp_conn; + unp_pcb_hold(unp); + if (__predict_false(unp == unp2)) { + unp_disconnect(unp, unp2); + } else if (unp2 != NULL) { + unp_pcb_hold(unp2); + unp_pcb_owned_lock2(unp, unp2, freed); + unp_disconnect(unp, unp2); + if (unp_pcb_rele(unp2) == 0) + UNP_PCB_UNLOCK(unp2); + } + if (unp_pcb_rele(unp) == 0) + UNP_PCB_UNLOCK(unp); #ifndef __rtems__ - if (vp) + if (vp) { + mtx_unlock(vplock); vrele(vp); + } #endif /* __rtems__ */ } @@ -734,17 +848,18 @@ uipc_connect2(struct socket *so1, struct socket *so2) struct unpcb *unp, *unp2; int error; - UNP_LINK_WLOCK(); unp = so1->so_pcb; KASSERT(unp != NULL, ("uipc_connect2: unp == NULL")); - UNP_PCB_LOCK(unp); unp2 = so2->so_pcb; KASSERT(unp2 != NULL, ("uipc_connect2: unp2 == NULL")); - UNP_PCB_LOCK(unp2); + if (unp != unp2) + unp_pcb_lock2(unp, unp2); + else + UNP_PCB_LOCK(unp); error = unp_connect2(so1, so2, PRU_CONNECT2); - UNP_PCB_UNLOCK(unp2); + if (unp != unp2) + UNP_PCB_UNLOCK(unp2); UNP_PCB_UNLOCK(unp); - UNP_LINK_WUNLOCK(); return (error); } @@ -752,6 +867,9 @@ static void uipc_detach(struct socket *so) { struct unpcb *unp, *unp2; +#ifndef __rtems__ + struct mtx *vplock; +#endif /* __rtems__ */ struct sockaddr_un *saved_unp_addr; #ifndef __rtems__ struct vnode *vp; @@ -766,6 +884,7 @@ uipc_detach(struct socket *so) vp = NULL; #ifndef __rtems__ + vplock = NULL; local_unp_rights = 0; #endif /* __rtems__ */ @@ -773,52 +892,87 @@ uipc_detach(struct socket *so) LIST_REMOVE(unp, unp_link); unp->unp_gencnt = ++unp_gencnt; --unp_count; + UNP_LINK_WUNLOCK(); + + UNP_PCB_UNLOCK_ASSERT(unp); + restart: + if ((vp = unp->unp_vnode) != NULL) { +#ifndef __rtems__ + vplock = mtx_pool_find(mtxpool_sleep, vp); + mtx_lock(vplock); +#endif /* __rtems__ */ + } UNP_PCB_LOCK(unp); - if ((unp->unp_flags & UNP_NASCENT) != 0) + if (unp->unp_vnode != vp && + unp->unp_vnode != NULL) { +#ifndef __rtems__ + if (vplock) + mtx_unlock(vplock); +#endif /* __rtems__ */ + UNP_PCB_UNLOCK(unp); + goto restart; + } + if ((unp->unp_flags & UNP_NASCENT) != 0) { goto teardown; - + } if ((vp = unp->unp_vnode) != NULL) { VOP_UNP_DETACH(vp); unp->unp_vnode = NULL; } - unp2 = unp->unp_conn; + if (__predict_false(unp == unp->unp_conn)) { + unp_disconnect(unp, unp); + unp2 = NULL; + goto connect_self; + } + if ((unp2 = unp->unp_conn) != NULL) { + unp_pcb_owned_lock2(unp, unp2, freeunp); + if (freeunp) + unp2 = NULL; + } + unp_pcb_hold(unp); if (unp2 != NULL) { - UNP_PCB_LOCK(unp2); + unp_pcb_hold(unp2); unp_disconnect(unp, unp2); - UNP_PCB_UNLOCK(unp2); + if (unp_pcb_rele(unp2) == 0) + UNP_PCB_UNLOCK(unp2); } - - /* - * We hold the linkage lock exclusively, so it's OK to acquire - * multiple pcb locks at a time. - */ + connect_self: + UNP_PCB_UNLOCK(unp); + UNP_REF_LIST_LOCK(); while (!LIST_EMPTY(&unp->unp_refs)) { struct unpcb *ref = LIST_FIRST(&unp->unp_refs); - UNP_PCB_LOCK(ref); + unp_pcb_hold(ref); + UNP_REF_LIST_UNLOCK(); + + MPASS(ref != unp); + UNP_PCB_UNLOCK_ASSERT(ref); unp_drop(ref); - UNP_PCB_UNLOCK(ref); + UNP_REF_LIST_LOCK(); } + + UNP_REF_LIST_UNLOCK(); + UNP_PCB_LOCK(unp); + freeunp = unp_pcb_rele(unp); + MPASS(freeunp == 0); #ifndef __rtems__ local_unp_rights = unp_rights; #endif /* __rtems__ */ teardown: - UNP_LINK_WUNLOCK(); unp->unp_socket->so_pcb = NULL; saved_unp_addr = unp->unp_addr; unp->unp_addr = NULL; - unp->unp_refcount--; - freeunp = (unp->unp_refcount == 0); + unp->unp_socket = NULL; + freeunp = unp_pcb_rele(unp); if (saved_unp_addr != NULL) free(saved_unp_addr, M_SONAME); - if (freeunp) { - UNP_PCB_LOCK_DESTROY(unp); - uma_zfree(unp_zone, unp); - } else + if (!freeunp) UNP_PCB_UNLOCK(unp); #ifndef __rtems__ - if (vp) + if (vp) { + mtx_unlock(vplock); vrele(vp); + } if (local_unp_rights) taskqueue_enqueue_timeout(taskqueue_thread, &unp_gc_task, -1); #endif /* __rtems__ */ @@ -828,20 +982,32 @@ static int uipc_disconnect(struct socket *so) { struct unpcb *unp, *unp2; + int freed; unp = sotounpcb(so); KASSERT(unp != NULL, ("uipc_disconnect: unp == NULL")); - UNP_LINK_WLOCK(); UNP_PCB_LOCK(unp); - unp2 = unp->unp_conn; - if (unp2 != NULL) { - UNP_PCB_LOCK(unp2); - unp_disconnect(unp, unp2); - UNP_PCB_UNLOCK(unp2); + if ((unp2 = unp->unp_conn) == NULL) { + UNP_PCB_UNLOCK(unp); + return (0); } - UNP_PCB_UNLOCK(unp); - UNP_LINK_WUNLOCK(); + if (unp == unp2) { + if (unp_pcb_rele(unp) == 0) + UNP_PCB_UNLOCK(unp); + } + unp_pcb_owned_lock2(unp, unp2, freed); + if (__predict_false(freed)) { + UNP_PCB_UNLOCK(unp); + return (0); + } + unp_pcb_hold(unp2); + unp_pcb_hold(unp); + unp_disconnect(unp, unp2); + if (unp_pcb_rele(unp) == 0) + UNP_PCB_UNLOCK(unp); + if (unp_pcb_rele(unp2) == 0) + UNP_PCB_UNLOCK(unp2); return (0); } @@ -960,13 +1126,35 @@ uipc_rcvd(struct socket *so, int flags) } static int +connect_internal(struct socket *so, struct sockaddr *nam, struct thread *td) +{ + int error; + struct unpcb *unp; + + unp = so->so_pcb; + if (unp->unp_conn != NULL) + return (EISCONN); + error = unp_connect(so, nam, td); + if (error) + return (error); + UNP_PCB_LOCK(unp); + if (unp->unp_conn == NULL) { + UNP_PCB_UNLOCK(unp); + if (error == 0) + error = ENOTCONN; + } + return (error); +} + + +static int uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam, struct mbuf *control, struct thread *td) { struct unpcb *unp, *unp2; struct socket *so2; u_int mbcnt, sbcc; - int error = 0; + int freed, error; unp = sotounpcb(so); KASSERT(unp != NULL, ("%s: unp == NULL", __func__)); @@ -974,6 +1162,7 @@ uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam, so->so_type == SOCK_SEQPACKET, ("%s: socktype %d", __func__, so->so_type)); + freed = error = 0; if (flags & PRUS_OOB) { error = EOPNOTSUPP; goto release; @@ -981,53 +1170,72 @@ uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam, #ifndef __rtems__ if (control != NULL && (error = unp_internalize(&control, td))) goto release; -#else /* __rtems__ */ - if (control != NULL) { - error = ENOSYS; - goto release; - } #endif /* __rtems__ */ - if ((nam != NULL) || (flags & PRUS_EOF)) - UNP_LINK_WLOCK(); - else - UNP_LINK_RLOCK(); + + unp2 = NULL; switch (so->so_type) { case SOCK_DGRAM: { const struct sockaddr *from; - unp2 = unp->unp_conn; if (nam != NULL) { - UNP_LINK_WLOCK_ASSERT(); - if (unp2 != NULL) { - error = EISCONN; + /* + * We return with UNP_PCB_LOCK_HELD so we know that + * the reference is live if the pointer is valid. + */ + if ((error = connect_internal(so, nam, td))) + break; + MPASS(unp->unp_conn != NULL); + unp2 = unp->unp_conn; + } else { + UNP_PCB_LOCK(unp); + + /* + * Because connect() and send() are non-atomic in a sendto() + * with a target address, it's possible that the socket will + * have disconnected before the send() can run. In that case + * return the slightly counter-intuitive but otherwise + * correct error that the socket is not connected. + */ + if ((unp2 = unp->unp_conn) == NULL) { + UNP_PCB_UNLOCK(unp); + error = ENOTCONN; break; } - error = unp_connect(so, nam, td); - if (error) + } + if (__predict_false(unp == unp2)) { + if (unp->unp_socket == NULL) { + error = ENOTCONN; break; - unp2 = unp->unp_conn; + } + goto connect_self; + } + unp_pcb_owned_lock2(unp, unp2, freed); + if (__predict_false(freed)) { + UNP_PCB_UNLOCK(unp); + error = ENOTCONN; + break; } - /* - * Because connect() and send() are non-atomic in a sendto() - * with a target address, it's possible that the socket will - * have disconnected before the send() can run. In that case - * return the slightly counter-intuitive but otherwise - * correct error that the socket is not connected. + * The socket referencing unp2 may have been closed + * or unp may have been disconnected if the unp lock + * was dropped to acquire unp2. */ - if (unp2 == NULL) { + if (__predict_false(unp->unp_conn == NULL) || + unp2->unp_socket == NULL) { + UNP_PCB_UNLOCK(unp); + if (unp_pcb_rele(unp2) == 0) + UNP_PCB_UNLOCK(unp2); error = ENOTCONN; break; } - /* Lockless read. */ + connect_self: if (unp2->unp_flags & UNP_WANTCRED) #ifndef __rtems__ control = unp_addsockcred(td, control); #else /* __rtems__ */ control = NULL; #endif /* __rtems__ */ - UNP_PCB_LOCK(unp); if (unp->unp_addr != NULL) from = (struct sockaddr *)unp->unp_addr; else @@ -1043,12 +1251,10 @@ uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam, SOCKBUF_UNLOCK(&so2->so_rcv); error = ENOBUFS; } - if (nam != NULL) { - UNP_LINK_WLOCK_ASSERT(); - UNP_PCB_LOCK(unp2); + if (nam != NULL) unp_disconnect(unp, unp2); + if (__predict_true(unp != unp2)) UNP_PCB_UNLOCK(unp2); - } UNP_PCB_UNLOCK(unp); break; } @@ -1057,42 +1263,37 @@ uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam, case SOCK_STREAM: if ((so->so_state & SS_ISCONNECTED) == 0) { if (nam != NULL) { - UNP_LINK_WLOCK_ASSERT(); - error = unp_connect(so, nam, td); - if (error) - break; /* XXX */ - } else { + if ((error = connect_internal(so, nam, td))) + break; + } else { error = ENOTCONN; break; } - } - - /* Lockless read. */ - if (so->so_snd.sb_state & SBS_CANTSENDMORE) { + } else if ((unp2 = unp->unp_conn) == NULL) { + error = ENOTCONN; + break; + } else if (so->so_snd.sb_state & SBS_CANTSENDMORE) { error = EPIPE; break; + } else { + UNP_PCB_LOCK(unp); + if ((unp2 = unp->unp_conn) == NULL) { + UNP_PCB_UNLOCK(unp); + error = ENOTCONN; + break; + } } - - /* - * Because connect() and send() are non-atomic in a sendto() - * with a target address, it's possible that the socket will - * have disconnected before the send() can run. In that case - * return the slightly counter-intuitive but otherwise - * correct error that the socket is not connected. - * - * Locking here must be done carefully: the linkage lock - * prevents interconnections between unpcbs from changing, so - * we can traverse from unp to unp2 without acquiring unp's - * lock. Socket buffer locks follow unpcb locks, so we can - * acquire both remote and lock socket buffer locks. - */ - unp2 = unp->unp_conn; - if (unp2 == NULL) { + unp_pcb_owned_lock2(unp, unp2, freed); + UNP_PCB_UNLOCK(unp); + if (__predict_false(freed)) { + error = ENOTCONN; + break; + } + if ((so2 = unp2->unp_socket) == NULL) { + UNP_PCB_UNLOCK(unp2); error = ENOTCONN; break; } - so2 = unp2->unp_socket; - UNP_PCB_LOCK(unp2); SOCKBUF_LOCK(&so2->so_rcv); if (unp2->unp_flags & UNP_WANTCRED) { #ifndef __rtems__ @@ -1167,12 +1368,6 @@ uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam, unp_shutdown(unp); UNP_PCB_UNLOCK(unp); } - - if ((nam != NULL) || (flags & PRUS_EOF)) - UNP_LINK_WUNLOCK(); - else - UNP_LINK_RUNLOCK(); - if (control != NULL && error != 0) #ifndef __rtems__ unp_dispose_mbuf(control); @@ -1249,12 +1444,10 @@ uipc_shutdown(struct socket *so) unp = sotounpcb(so); KASSERT(unp != NULL, ("uipc_shutdown: unp == NULL")); - UNP_LINK_WLOCK(); UNP_PCB_LOCK(unp); socantsendmore(so); unp_shutdown(unp); UNP_PCB_UNLOCK(unp); - UNP_LINK_WUNLOCK(); return (0); } @@ -1470,16 +1663,13 @@ unp_connectat(int fd, struct socket *so, struct sockaddr *nam, #ifndef __rtems__ cap_rights_t rights; #endif /* __rtems__ */ - int error, len; + int error, len, freed; +#ifndef __rtems__ + struct mtx *vplock; +#endif /* __rtems__ */ if (nam->sa_family != AF_UNIX) return (EAFNOSUPPORT); - - UNP_LINK_WLOCK_ASSERT(); - - unp = sotounpcb(so); - KASSERT(unp != NULL, ("unp_connect: unp == NULL")); - if (nam->sa_len > sizeof(struct sockaddr_un)) return (EINVAL); len = nam->sa_len - offsetof(struct sockaddr_un, sun_path); @@ -1490,12 +1680,12 @@ unp_connectat(int fd, struct socket *so, struct sockaddr *nam, buf[len] = 0; #endif /* __rtems__ */ + unp = sotounpcb(so); UNP_PCB_LOCK(unp); if (unp->unp_flags & UNP_CONNECTING) { UNP_PCB_UNLOCK(unp); return (EALREADY); } - UNP_LINK_WUNLOCK(); unp->unp_flags |= UNP_CONNECTING; UNP_PCB_UNLOCK(unp); @@ -1548,12 +1738,9 @@ unp_connectat(int fd, struct socket *so, struct sockaddr *nam, unp = sotounpcb(so); KASSERT(unp != NULL, ("unp_connect: unp == NULL")); - /* - * Lock linkage lock for two reasons: make sure v_socket is stable, - * and to protect simultaneous locking of multiple pcbs. - */ - UNP_LINK_WLOCK(); #ifndef __rtems__ + vplock = mtx_pool_find(mtxpool_sleep, vp); + mtx_lock(vplock); VOP_UNP_CONNECT(vp, &unp2); if (unp2 == NULL) { error = ECONNREFUSED; @@ -1572,8 +1759,6 @@ unp_connectat(int fd, struct socket *so, struct sockaddr *nam, error = EPROTOTYPE; goto bad2; } - UNP_PCB_LOCK(unp); - UNP_PCB_LOCK(unp2); if (so->so_proto->pr_flags & PR_CONNREQUIRED) { if (so2->so_options & SO_ACCEPTCONN) { CURVNET_SET(so2->so_vnet); @@ -1583,10 +1768,10 @@ unp_connectat(int fd, struct socket *so, struct sockaddr *nam, so2 = NULL; if (so2 == NULL) { error = ECONNREFUSED; - goto bad3; + goto bad2; } unp3 = sotounpcb(so2); - UNP_PCB_LOCK(unp3); + unp_pcb_lock2(unp2, unp3); if (unp2->unp_addr != NULL) { bcopy(unp2->unp_addr, sa, unp2->unp_addr->sun_len); unp3->unp_addr = (struct sockaddr_un *) sa; @@ -1613,28 +1798,40 @@ unp_connectat(int fd, struct socket *so, struct sockaddr *nam, unp3->unp_flags |= UNP_WANTCRED; UNP_PCB_UNLOCK(unp2); unp2 = unp3; + unp_pcb_owned_lock2(unp2, unp, freed); + if (__predict_false(freed)) { + UNP_PCB_UNLOCK(unp2); + error = ECONNREFUSED; + goto bad2; + } #ifdef MAC mac_socketpeer_set_from_socket(so, so2); mac_socketpeer_set_from_socket(so2, so); #endif + } else { + if (unp == unp2) + UNP_PCB_LOCK(unp); + else + unp_pcb_lock2(unp, unp2); } - KASSERT(unp2 != NULL && so2 != NULL && unp2->unp_socket == so2 && sotounpcb(so2) == unp2, ("%s: unp2 %p so2 %p", __func__, unp2, so2)); error = unp_connect2(so, so2, PRU_CONNECT); -bad3: - UNP_PCB_UNLOCK(unp2); + if (unp != unp2) + UNP_PCB_UNLOCK(unp2); UNP_PCB_UNLOCK(unp); bad2: - UNP_LINK_WUNLOCK(); +#ifndef __rtems__ + mtx_unlock(vplock); +#endif /* __rtems__ */ bad: #ifndef __rtems__ - if (vp != NULL) + if (vp != NULL) { vput(vp); + } #endif /* __rtems__ */ free(sa, M_SONAME); - UNP_LINK_WLOCK(); UNP_PCB_LOCK(unp); unp->unp_flags &= ~UNP_CONNECTING; UNP_PCB_UNLOCK(unp); @@ -1652,7 +1849,6 @@ unp_connect2(struct socket *so, struct socket *so2, int req) unp2 = sotounpcb(so2); KASSERT(unp2 != NULL, ("unp_connect2: unp2 == NULL")); - UNP_LINK_WLOCK_ASSERT(); UNP_PCB_LOCK_ASSERT(unp); UNP_PCB_LOCK_ASSERT(unp2); @@ -1660,10 +1856,13 @@ unp_connect2(struct socket *so, struct socket *so2, int req) return (EPROTOTYPE); unp2->unp_flags &= ~UNP_NASCENT; unp->unp_conn = unp2; - + unp_pcb_hold(unp2); + unp_pcb_hold(unp); switch (so->so_type) { case SOCK_DGRAM: + UNP_REF_LIST_LOCK(); LIST_INSERT_HEAD(&unp2->unp_refs, unp, unp_reflink); + UNP_REF_LIST_UNLOCK(); soisconnected(so); break; @@ -1687,31 +1886,47 @@ unp_connect2(struct socket *so, struct socket *so2, int req) static void unp_disconnect(struct unpcb *unp, struct unpcb *unp2) { - struct socket *so; + struct socket *so, *so2; + int freed __unused; KASSERT(unp2 != NULL, ("unp_disconnect: unp2 == NULL")); - UNP_LINK_WLOCK_ASSERT(); UNP_PCB_LOCK_ASSERT(unp); UNP_PCB_LOCK_ASSERT(unp2); + if (unp->unp_conn == NULL && unp2->unp_conn == NULL) + return; + + MPASS(unp->unp_conn == unp2); unp->unp_conn = NULL; + so = unp->unp_socket; + so2 = unp2->unp_socket; switch (unp->unp_socket->so_type) { case SOCK_DGRAM: + UNP_REF_LIST_LOCK(); LIST_REMOVE(unp, unp_reflink); - so = unp->unp_socket; - SOCK_LOCK(so); - so->so_state &= ~SS_ISCONNECTED; - SOCK_UNLOCK(so); + UNP_REF_LIST_UNLOCK(); + if (so) { + SOCK_LOCK(so); + so->so_state &= ~SS_ISCONNECTED; + SOCK_UNLOCK(so); + } break; case SOCK_STREAM: case SOCK_SEQPACKET: - soisdisconnected(unp->unp_socket); + if (so) + soisdisconnected(so); + MPASS(unp2->unp_conn == unp); unp2->unp_conn = NULL; - soisdisconnected(unp2->unp_socket); + if (so2) + soisdisconnected(so2); break; } + freed = unp_pcb_rele(unp); + MPASS(freed == 0); + freed = unp_pcb_rele(unp2); + MPASS(freed == 0); } /* @@ -1795,7 +2010,7 @@ unp_pcblist(SYSCTL_HANDLER_ARGS) continue; } unp_list[i++] = unp; - unp->unp_refcount++; + unp_pcb_hold(unp); } UNP_PCB_UNLOCK(unp); } @@ -1807,8 +2022,9 @@ unp_pcblist(SYSCTL_HANDLER_ARGS) for (i = 0; i < n; i++) { unp = unp_list[i]; UNP_PCB_LOCK(unp); - unp->unp_refcount--; - if (unp->unp_refcount != 0 && unp->unp_gencnt <= gencnt) { + freeunp = unp_pcb_rele(unp); + + if (freeunp == 0 && unp->unp_gencnt <= gencnt) { xu->xu_len = sizeof *xu; xu->xu_unpp = unp; /* @@ -1835,14 +2051,8 @@ unp_pcblist(SYSCTL_HANDLER_ARGS) sotoxsocket(unp->unp_socket, &xu->xu_socket); UNP_PCB_UNLOCK(unp); error = SYSCTL_OUT(req, xu, sizeof *xu); - } else { - freeunp = (unp->unp_refcount == 0); + } else if (freeunp == 0) UNP_PCB_UNLOCK(unp); - if (freeunp) { - UNP_PCB_LOCK_DESTROY(unp); - uma_zfree(unp_zone, unp); - } - } } free(xu, M_TEMP); if (!error) { @@ -1879,7 +2089,6 @@ unp_shutdown(struct unpcb *unp) struct unpcb *unp2; struct socket *so; - UNP_LINK_WLOCK_ASSERT(); UNP_PCB_LOCK_ASSERT(unp); unp2 = unp->unp_conn; @@ -1896,22 +2105,30 @@ unp_drop(struct unpcb *unp) { struct socket *so = unp->unp_socket; struct unpcb *unp2; - - UNP_LINK_WLOCK_ASSERT(); - UNP_PCB_LOCK_ASSERT(unp); + int freed; /* * Regardless of whether the socket's peer dropped the connection * with this socket by aborting or disconnecting, POSIX requires * that ECONNRESET is returned. */ - so->so_error = ECONNRESET; + /* acquire a reference so that unp isn't freed from underneath us */ + + UNP_PCB_LOCK(unp); + if (so) + so->so_error = ECONNRESET; unp2 = unp->unp_conn; - if (unp2 == NULL) - return; - UNP_PCB_LOCK(unp2); - unp_disconnect(unp, unp2); - UNP_PCB_UNLOCK(unp2); + if (unp2 == unp) { + unp_disconnect(unp, unp2); + } else if (unp2 != NULL) { + unp_pcb_hold(unp2); + unp_pcb_owned_lock2(unp, unp2, freed); + unp_disconnect(unp, unp2); + if (unp_pcb_rele(unp2) == 0) + UNP_PCB_UNLOCK(unp2); + } + if (unp_pcb_rele(unp) == 0) + UNP_PCB_UNLOCK(unp); } #ifndef __rtems__ @@ -2053,7 +2270,7 @@ unp_init(void) return; #endif unp_zone = uma_zcreate("unpcb", sizeof(struct unpcb), NULL, NULL, - NULL, NULL, UMA_ALIGN_PTR, 0); + NULL, NULL, UMA_ALIGN_CACHE, 0); if (unp_zone == NULL) panic("unp_init"); uma_zone_set_max(unp_zone, maxsockets); @@ -2639,13 +2856,15 @@ vfs_unp_reclaim(struct vnode *vp) { struct unpcb *unp; int active; + struct mtx *vplock; ASSERT_VOP_ELOCKED(vp, "vfs_unp_reclaim"); KASSERT(vp->v_type == VSOCK, ("vfs_unp_reclaim: vp->v_type != VSOCK")); active = 0; - UNP_LINK_WLOCK(); + vplock = mtx_pool_find(mtxpool_sleep, vp); + mtx_lock(vplock); VOP_UNP_CONNECT(vp, &unp); if (unp == NULL) goto done; @@ -2656,8 +2875,8 @@ vfs_unp_reclaim(struct vnode *vp) active = 1; } UNP_PCB_UNLOCK(unp); -done: - UNP_LINK_WUNLOCK(); + done: + mtx_unlock(vplock); if (active) vunref(vp); } diff --git a/freebsd/sys/mips/include/machine/cpufunc.h b/freebsd/sys/mips/include/machine/cpufunc.h index a83d22bf..7cfc548c 100644 --- a/freebsd/sys/mips/include/machine/cpufunc.h +++ b/freebsd/sys/mips/include/machine/cpufunc.h @@ -108,6 +108,12 @@ mips_wbflush(void) #endif } +static __inline void +breakpoint(void) +{ + __asm __volatile ("break"); +} + #ifdef _KERNEL /* * XXX @@ -279,6 +285,8 @@ MIPS_RW32_COP0(entrylo0, MIPS_COP_0_TLB_LO0); MIPS_RW32_COP0(entrylo1, MIPS_COP_0_TLB_LO1); #endif MIPS_RW32_COP0(prid, MIPS_COP_0_PRID); +MIPS_RW32_COP0_SEL(cinfo, MIPS_COP_0_PRID, 6); +MIPS_RW32_COP0_SEL(tinfo, MIPS_COP_0_PRID, 7); /* XXX 64-bit? */ MIPS_RW32_COP0_SEL(ebase, MIPS_COP_0_PRID, 1); @@ -363,12 +371,6 @@ get_intr_mask(void) return (mips_rd_status() & MIPS_SR_INT_MASK); } -static __inline void -breakpoint(void) -{ - __asm __volatile ("break"); -} - #if defined(__GNUC__) && !defined(__mips_o32) #define mips3_ld(a) (*(const volatile uint64_t *)(a)) #define mips3_sd(a, v) (*(volatile uint64_t *)(a) = (v)) diff --git a/freebsd/sys/net/altq/altq_subr.c b/freebsd/sys/net/altq/altq_subr.c index 6a385560..47a353fc 100644 --- a/freebsd/sys/net/altq/altq_subr.c +++ b/freebsd/sys/net/altq/altq_subr.c @@ -436,8 +436,8 @@ tbr_timeout(arg) VNET_LIST_RLOCK_NOSLEEP(); VNET_FOREACH(vnet_iter) { CURVNET_SET(vnet_iter); - for (ifp = TAILQ_FIRST(&V_ifnet); ifp; - ifp = TAILQ_NEXT(ifp, if_link)) { + for (ifp = CK_STAILQ_FIRST(&V_ifnet); ifp; + ifp = CK_STAILQ_NEXT(ifp, if_link)) { /* read from if_snd unlocked */ if (!TBR_IS_ENABLED(&ifp->if_snd)) continue; diff --git a/freebsd/sys/net/bpf.c b/freebsd/sys/net/bpf.c index 24927e8b..57aff5b8 100644 --- a/freebsd/sys/net/bpf.c +++ b/freebsd/sys/net/bpf.c @@ -42,7 +42,6 @@ __FBSDID("$FreeBSD$"); #include <rtems/bsd/local/opt_bpf.h> -#include <rtems/bsd/local/opt_compat.h> #include <rtems/bsd/local/opt_ddb.h> #include <rtems/bsd/local/opt_netgraph.h> @@ -106,6 +105,10 @@ __FBSDID("$FreeBSD$"); MALLOC_DEFINE(M_BPF, "BPF", "BPF data"); +static struct bpf_if_ext dead_bpf_if = { + .bif_dlist = LIST_HEAD_INITIALIZER() +}; + struct bpf_if { #define bif_next bif_ext.bif_next #define bif_dlist bif_ext.bif_dlist @@ -167,6 +170,9 @@ struct bpf_dltlist32 { #define BIOCSETFNR32 _IOW('B', 130, struct bpf_program32) #endif +#define BPF_LOCK() sx_xlock(&bpf_sx) +#define BPF_UNLOCK() sx_xunlock(&bpf_sx) +#define BPF_LOCK_ASSERT() sx_assert(&bpf_sx, SA_XLOCKED) /* * bpf_iflist is a list of BPF interface structures, each corresponding to a * specific DLT. The same network interface might have several BPF interface @@ -174,7 +180,7 @@ struct bpf_dltlist32 { * frames, ethernet frames, etc). */ static LIST_HEAD(, bpf_if) bpf_iflist, bpf_freelist; -static struct mtx bpf_mtx; /* bpf global lock */ +static struct sx bpf_sx; /* bpf global lock */ static int bpf_bpfd_cnt; static void bpf_attachd(struct bpf_d *, struct bpf_if *); @@ -2752,7 +2758,7 @@ bpfdetach(struct ifnet *ifp) */ BPFIF_WLOCK(bp); bp->bif_flags |= BPFIF_FLAG_DYING; - *bp->bif_bpf = NULL; + *bp->bif_bpf = (struct bpf_if *)&dead_bpf_if; BPFIF_WUNLOCK(bp); CTR4(KTR_NET, "%s: sheduling free for encap %d (%p) for if %p", @@ -3090,7 +3096,7 @@ bpf_drvinit(void *unused) int rv; #endif /* __rtems__ */ - mtx_init(&bpf_mtx, "bpf global lock", NULL, MTX_DEF); + sx_init(&bpf_sx, "bpf global lock"); LIST_INIT(&bpf_iflist); LIST_INIT(&bpf_freelist); @@ -3253,13 +3259,13 @@ bpf_stats_sysctl(SYSCTL_HANDLER_ARGS) SYSINIT(bpfdev,SI_SUB_DRIVERS,SI_ORDER_MIDDLE,bpf_drvinit,NULL); #else /* !DEV_BPF && !NETGRAPH_BPF */ + /* * NOP stubs to allow bpf-using drivers to load and function. * * A 'better' implementation would allow the core bpf functionality * to be loaded at runtime. */ -static struct bpf_if bp_null; void bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) @@ -3287,7 +3293,7 @@ void bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) { - *driverp = &bp_null; + *driverp = (struct bpf_if *)&dead_bpf_if; } void diff --git a/freebsd/sys/net/bpfdesc.h b/freebsd/sys/net/bpfdesc.h index 695d3d40..95093cff 100644 --- a/freebsd/sys/net/bpfdesc.h +++ b/freebsd/sys/net/bpfdesc.h @@ -125,9 +125,6 @@ struct bpf_d { #define BPF_PID_REFRESH_CUR(bd) do { } while (0) #endif /* __rtems__ */ -#define BPF_LOCK() mtx_lock(&bpf_mtx) -#define BPF_UNLOCK() mtx_unlock(&bpf_mtx) -#define BPF_LOCK_ASSERT() mtx_assert(&bpf_mtx, MA_OWNED) /* * External representation of the bpf descriptor */ diff --git a/freebsd/sys/net/bridgestp.c b/freebsd/sys/net/bridgestp.c index 472b9634..49e772b3 100644 --- a/freebsd/sys/net/bridgestp.c +++ b/freebsd/sys/net/bridgestp.c @@ -2045,7 +2045,7 @@ bstp_reinit(struct bstp_state *bs) * bridges in the same STP domain. */ IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (ifp->if_type != IFT_ETHER) continue; /* Not Ethernet */ diff --git a/freebsd/sys/net/dlt.h b/freebsd/sys/net/dlt.h index dc818521..639e5a7f 100644 --- a/freebsd/sys/net/dlt.h +++ b/freebsd/sys/net/dlt.h @@ -122,9 +122,9 @@ /* * 18 is used for DLT_PFSYNC in OpenBSD, NetBSD, DragonFly BSD and - * Mac OS X; don't use it for anything else. (FreeBSD uses 121, - * which collides with DLT_HHDLC, even though it doesn't use 18 - * for anything and doesn't appear to have ever used it for anything.) + * macOS; don't use it for anything else. (FreeBSD uses 121, which + * collides with DLT_HHDLC, even though it doesn't use 18 for + * anything and doesn't appear to have ever used it for anything.) * * We define it as 18 on those platforms; it is, unfortunately, used * for DLT_CIP in Suse 6.3, so we don't define it as DLT_PFSYNC @@ -342,7 +342,7 @@ * * FreeBSD's libpcap won't map a link-layer header type of 18 - i.e., * DLT_PFSYNC files from OpenBSD and possibly older versions of NetBSD, - * DragonFly BSD, and OS X - to DLT_PFSYNC, so code built with FreeBSD's + * DragonFly BSD, and macOS - to DLT_PFSYNC, so code built with FreeBSD's * libpcap won't treat those files as DLT_PFSYNC files. * * Other libpcaps won't map a link-layer header type of 121 to DLT_PFSYNC; @@ -948,14 +948,14 @@ * the pseudo-header is: * * struct dl_ipnetinfo { - * u_int8_t dli_version; - * u_int8_t dli_family; - * u_int16_t dli_htype; - * u_int32_t dli_pktlen; - * u_int32_t dli_ifindex; - * u_int32_t dli_grifindex; - * u_int32_t dli_zsrc; - * u_int32_t dli_zdst; + * uint8_t dli_version; + * uint8_t dli_family; + * uint16_t dli_htype; + * uint32_t dli_pktlen; + * uint32_t dli_ifindex; + * uint32_t dli_grifindex; + * uint32_t dli_zsrc; + * uint32_t dli_zdst; * }; * * dli_version is 2 for the current version of the pseudo-header. @@ -1231,7 +1231,7 @@ * So I'll just give them one; hopefully this will show up in a * libpcap release in time for them to get this into 10.10 Big Sur * or whatever Mavericks' successor is called. LINKTYPE_PKTAP - * will be 258 *even on OS X*; that is *intentional*, so that + * will be 258 *even on macOS*; that is *intentional*, so that * PKTAP files look the same on *all* OSes (different OSes can have * different numerical values for a given DLT_, but *MUST NOT* have * different values for what goes in a file, as files can be moved @@ -1243,9 +1243,9 @@ * and that will continue to be DLT_USER2 on Darwin-based OSes. That way, * binary compatibility with Mavericks is preserved for programs using * this version of libpcap. This does mean that if you were using - * DLT_USER2 for some capture device on OS X, you can't do so with + * DLT_USER2 for some capture device on macOS, you can't do so with * this version of libpcap, just as you can't with Apple's libpcap - - * on OS X, they define DLT_PKTAP to be DLT_USER2, so programs won't + * on macOS, they define DLT_PKTAP to be DLT_USER2, so programs won't * be able to distinguish between PKTAP and whatever you were using * DLT_USER2 for. * @@ -1305,6 +1305,66 @@ #define DLT_RDS 265 /* + * USB packets, beginning with a Darwin (macOS, etc.) header. + */ +#define DLT_USB_DARWIN 266 + +/* + * OpenBSD DLT_OPENFLOW. + */ +#define DLT_OPENFLOW 267 + +/* + * SDLC frames containing SNA PDUs. + */ +#define DLT_SDLC 268 + +/* + * per "Selvig, Bjorn" <b.selvig@ti.com> used for + * TI protocol sniffer. + */ +#define DLT_TI_LLN_SNIFFER 269 + +/* + * per: Erik de Jong <erikdejong at gmail.com> for + * https://github.com/eriknl/LoRaTap/releases/tag/v0.1 + */ +#define DLT_LORATAP 270 + +/* + * per: Stefanha at gmail.com for + * http://lists.sandelman.ca/pipermail/tcpdump-workers/2017-May/000772.html + * and: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/uapi/linux/vsockmon.h + * for: http://qemu-project.org/Features/VirtioVsock + */ +#define DLT_VSOCK 271 + +/* + * Nordic Semiconductor Bluetooth LE sniffer. + */ +#define DLT_NORDIC_BLE 272 + +/* + * Excentis DOCSIS 3.1 RF sniffer (XRA-31) + * per: bruno.verstuyft at excentis.com + * http://www.xra31.com/xra-header + */ +#define DLT_DOCSIS31_XRA31 273 + +/* + * mPackets, as specified by IEEE 802.3br Figure 99-4, starting + * with the preamble and always ending with a CRC field. + */ +#define DLT_ETHERNET_MPACKET 274 + +/* + * DisplayPort AUX channel monitoring data as specified by VESA + * DisplayPort(DP) Standard preceeded by a pseudo-header. + * per dirk.eibach at gdsys.cc + */ +#define DLT_DISPLAYPORT_AUX 275 + +/* * In case the code that includes this file (directly or indirectly) * has also included OS files that happen to define DLT_MATCHING_MAX, * with a different value (perhaps because that OS hasn't picked up @@ -1314,7 +1374,7 @@ #ifdef DLT_MATCHING_MAX #undef DLT_MATCHING_MAX #endif -#define DLT_MATCHING_MAX 265 /* highest value in the "matching" range */ +#define DLT_MATCHING_MAX 275 /* highest value in the "matching" range */ /* * DLT and savefile link type values are split into a class and diff --git a/freebsd/sys/net/fddi.h b/freebsd/sys/net/fddi.h deleted file mode 100644 index 0badcc3c..00000000 --- a/freebsd/sys/net/fddi.h +++ /dev/null @@ -1,107 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-4-Clause - * - * Copyright (c) 1982, 1986, 1993 - * The Regents of the University of California. All rights reserved. - * Copyright (c) 1995 Matt Thomas (thomas@lkg.dec.com) - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the University of - * California, Berkeley and its contributors. - * 4. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * @(#)if_fddi.h 8.1 (Berkeley) 6/10/93 - * $FreeBSD$ - */ - -#ifndef _NETINET_IF_FDDI_H_ -#define _NETINET_IF_FDDI_H_ - -#define FDDIIPMTU 4352 -#define FDDIMTU 4470 -#define FDDIMIN 3 - -#define FDDIFC_C 0x80 /* 0b10000000 */ -#define FDDIFC_L 0x40 /* 0b01000000 */ -#define FDDIFC_F 0x30 /* 0b00110000 */ -#define FDDIFC_Z 0x0F /* 0b00001111 */ -#define FDDIFC_CLFF 0xF0 /* Class/Length/Format bits */ -#define FDDIFC_ZZZZ 0x0F /* Control bits */ - -/* - * FDDI Frame Control values. (48-bit addressing only). - */ -#define FDDIFC_VOID 0x40 /* Void frame */ -#define FDDIFC_NRT 0x80 /* Nonrestricted token */ -#define FDDIFC_RT 0xc0 /* Restricted token */ -#define FDDIFC_MAC_BEACON 0xc2 /* MAC Beacon frame */ -#define FDDIFC_MAC_CLAIM 0xc3 /* MAC Claim frame */ -#define FDDIFC_LLC_ASYNC 0x50 -#define FDDIFC_LLC_PRIO0 0 -#define FDDIFC_LLC_PRIO1 1 -#define FDDIFC_LLC_PRIO2 2 -#define FDDIFC_LLC_PRIO3 3 -#define FDDIFC_LLC_PRIO4 4 -#define FDDIFC_LLC_PRIO5 5 -#define FDDIFC_LLC_PRIO6 6 -#define FDDIFC_LLC_PRIO7 7 -#define FDDIFC_LLC_SYNC 0xd0 -#define FDDIFC_IMP_ASYNC 0x60 /* Implementor Async. */ -#define FDDIFC_IMP_SYNC 0xe0 /* Implementor Synch. */ -#define FDDIFC_SMT 0x40 -#define FDDIFC_SMT_INFO 0x41 /* SMT Info */ -#define FDDIFC_SMT_NSA 0x4F /* SMT Next station adrs */ -#define FDDIFC_MAC 0xc0 /* MAC frame */ - -#define FDDI_ADDR_LEN 6 -#define FDDI_HDR_LEN (sizeof(struct fddi_header)) - -/* - * Structure of an 100Mb/s FDDI header. - */ -struct fddi_header { - u_char fddi_fc; - u_char fddi_dhost[FDDI_ADDR_LEN]; - u_char fddi_shost[FDDI_ADDR_LEN]; -}; - -#if defined(_KERNEL) -#define fddi_ipmulticast_min ether_ipmulticast_min -#define fddi_ipmulticast_max ether_ipmulticast_max -#define fddi_addmulti ether_addmulti -#define fddi_delmulti ether_delmulti -#define fddi_sprintf ether_sprintf - -#define FDDI_BPF_UNSUPPORTED 0 -#define FDDI_BPF_SUPPORTED 1 - -void fddi_ifattach(struct ifnet *, const u_int8_t *, int); -void fddi_ifdetach(struct ifnet *, int); -int fddi_ioctl(struct ifnet *, u_long, caddr_t); - -#endif /* _KERNEL */ -#endif /* _NET_FDDI_H_ */ diff --git a/freebsd/sys/net/if.c b/freebsd/sys/net/if.c index bbf2ddd0..d4c18b46 100644 --- a/freebsd/sys/net/if.c +++ b/freebsd/sys/net/if.c @@ -34,7 +34,6 @@ * $FreeBSD$ */ -#include <rtems/bsd/local/opt_compat.h> #include <rtems/bsd/local/opt_inet6.h> #include <rtems/bsd/local/opt_inet.h> @@ -44,6 +43,7 @@ #include <sys/malloc.h> #include <sys/sbuf.h> #include <sys/bus.h> +#include <sys/epoch.h> #include <sys/mbuf.h> #include <sys/systm.h> #include <sys/priv.h> @@ -90,6 +90,7 @@ #include <netinet/ip_carp.h> #ifdef INET #include <netinet/if_ether.h> +#include <netinet/netdump/netdump.h> #endif /* INET */ #ifdef INET6 #include <netinet6/in6_var.h> @@ -106,6 +107,13 @@ _Static_assert(sizeof(((struct ifreq *)0)->ifr_name) == offsetof(struct ifreq, ifr_ifru), "gap between ifr_name and ifr_ifru"); +#ifndef __rtems__ +__read_mostly epoch_t net_epoch_preempt; +__read_mostly epoch_t net_epoch; +#else /* __rtems__ */ +EPOCH_DEFINE(_bsd_net_epoch_preempt, EPOCH_PREEMPT); +EPOCH_DEFINE(_bsd_net_epoch, 0); +#endif /* __rtems__ */ #ifdef COMPAT_FREEBSD32 #include <sys/mount.h> #include <compat/freebsd32/freebsd32.h> @@ -144,7 +152,37 @@ struct ifreq32 { CTASSERT(sizeof(struct ifreq) == sizeof(struct ifreq32)); CTASSERT(__offsetof(struct ifreq, ifr_ifru) == __offsetof(struct ifreq32, ifr_ifru)); -#endif + +struct ifgroupreq32 { + char ifgr_name[IFNAMSIZ]; + u_int ifgr_len; + union { + char ifgru_group[IFNAMSIZ]; + uint32_t ifgru_groups; + } ifgr_ifgru; +}; + +struct ifmediareq32 { + char ifm_name[IFNAMSIZ]; + int ifm_current; + int ifm_mask; + int ifm_status; + int ifm_active; + int ifm_count; + uint32_t ifm_ulist; /* (int *) */ +}; +#define SIOCGIFMEDIA32 _IOC_NEWTYPE(SIOCGIFMEDIA, struct ifmediareq32) +#define SIOCGIFXMEDIA32 _IOC_NEWTYPE(SIOCGIFXMEDIA, struct ifmediareq32) + +#define _CASE_IOC_IFGROUPREQ_32(cmd) \ + case _IOC_NEWTYPE((cmd), struct ifgroupreq32): +#else /* !COMPAT_FREEBSD32 */ +#define _CASE_IOC_IFGROUPREQ_32(cmd) +#endif /* !COMPAT_FREEBSD32 */ + +#define CASE_IOC_IFGROUPREQ(cmd) \ + _CASE_IOC_IFGROUPREQ_32(cmd) \ + case (cmd) union ifreq_union { struct ifreq ifr; @@ -153,6 +191,13 @@ union ifreq_union { #endif }; +union ifgroupreq_union { + struct ifgroupreq ifgr; +#ifdef COMPAT_FREEBSD32 + struct ifgroupreq32 ifgr32; +#endif +}; + SYSCTL_NODE(_net, PF_LINK, link, CTLFLAG_RW, 0, "Link layers"); SYSCTL_NODE(_net_link, 0, generic, CTLFLAG_RW, 0, "Generic link-management"); @@ -185,7 +230,6 @@ static MALLOC_DEFINE(M_IFDESCR, "ifdescr", "ifnet descriptions"); static struct sx ifdescr_sx; SX_SYSINIT(ifdescr_sx, &ifdescr_sx, "ifnet descr"); -void (*bridge_linkstate_p)(struct ifnet *ifp); void (*ng_ether_link_state_p)(struct ifnet *ifp, int state); void (*lagg_linkstate_p)(struct ifnet *ifp, int state); /* These are external hooks for CARP. */ @@ -219,8 +263,7 @@ struct mbuf *(*tbr_dequeue_ptr)(struct ifaltq *, int) = NULL; static void if_attachdomain(void *); static void if_attachdomain1(struct ifnet *); static int ifconf(u_long, caddr_t); -static void if_freemulti(struct ifmultiaddr *); -static void if_grow(void); +static void *if_grow(void); static void if_input_default(struct ifnet *, struct mbuf *); static int if_requestencap_default(struct ifnet *, struct if_encap_req *); static void if_route(struct ifnet *, int flag, int fam); @@ -309,9 +352,7 @@ ifnet_byindex(u_short idx) { struct ifnet *ifp; - IFNET_RLOCK_NOSLEEP(); ifp = ifnet_byindex_locked(idx); - IFNET_RUNLOCK_NOSLEEP(); return (ifp); } @@ -336,12 +377,11 @@ ifnet_byindex_ref(u_short idx) * failure. */ static u_short -ifindex_alloc(void) +ifindex_alloc(void **old) { u_short idx; IFNET_WLOCK_ASSERT(); -retry: /* * Try to find an empty slot below V_if_index. If we fail, take the * next slot. @@ -353,8 +393,8 @@ retry: /* Catch if_index overflow. */ if (idx >= V_if_indexlim) { - if_grow(); - goto retry; + *old = if_grow(); + return (USHRT_MAX); } if (idx > V_if_index) V_if_index = idx; @@ -383,21 +423,10 @@ ifindex_free(u_short idx) } static void -ifnet_setbyindex_locked(u_short idx, struct ifnet *ifp) -{ - - IFNET_WLOCK_ASSERT(); - - V_ifindex_table[idx] = ifp; -} - -static void ifnet_setbyindex(u_short idx, struct ifnet *ifp) { - IFNET_WLOCK(); - ifnet_setbyindex_locked(idx, ifp); - IFNET_WUNLOCK(); + V_ifindex_table[idx] = ifp; } struct ifaddr * @@ -424,12 +453,15 @@ ifaddr_byindex(u_short idx) static void vnet_if_init(const void *unused __unused) { + void *old; - TAILQ_INIT(&V_ifnet); - TAILQ_INIT(&V_ifg_head); + CK_STAILQ_INIT(&V_ifnet); + CK_STAILQ_INIT(&V_ifg_head); IFNET_WLOCK(); - if_grow(); /* create initial table */ + old = if_grow(); /* create initial table */ IFNET_WUNLOCK(); + epoch_wait_preempt(net_epoch_preempt); + free(old, M_IFNET); vnet_if_clone_init(); } VNET_SYSINIT(vnet_if_init, SI_SUB_INIT_IF, SI_ORDER_SECOND, vnet_if_init, @@ -440,9 +472,9 @@ static void vnet_if_uninit(const void *unused __unused) { - VNET_ASSERT(TAILQ_EMPTY(&V_ifnet), ("%s:%d tailq &V_ifnet=%p " + VNET_ASSERT(CK_STAILQ_EMPTY(&V_ifnet), ("%s:%d tailq &V_ifnet=%p " "not empty", __func__, __LINE__, &V_ifnet)); - VNET_ASSERT(TAILQ_EMPTY(&V_ifg_head), ("%s:%d tailq &V_ifg_head=%p " + VNET_ASSERT(CK_STAILQ_EMPTY(&V_ifg_head), ("%s:%d tailq &V_ifg_head=%p " "not empty", __func__, __LINE__, &V_ifg_head)); free((caddr_t)V_ifindex_table, M_IFNET); @@ -456,7 +488,7 @@ vnet_if_return(const void *unused __unused) struct ifnet *ifp, *nifp; /* Return all inherited interfaces to their parent vnets. */ - TAILQ_FOREACH_SAFE(ifp, &V_ifnet, if_link, nifp) { + CK_STAILQ_FOREACH_SAFE(ifp, &V_ifnet, if_link, nifp) { if (ifp->if_home_vnet != ifp->if_vnet) if_vmove(ifp, ifp->if_home_vnet); } @@ -465,13 +497,16 @@ VNET_SYSUNINIT(vnet_if_return, SI_SUB_VNET_DONE, SI_ORDER_ANY, vnet_if_return, NULL); #endif -static void + +static void * if_grow(void) { int oldlim; u_int n; struct ifnet **e; + void *old; + old = NULL; IFNET_WLOCK_ASSERT(); oldlim = V_if_indexlim; IFNET_WUNLOCK(); @@ -480,14 +515,15 @@ if_grow(void) IFNET_WLOCK(); if (V_if_indexlim != oldlim) { free(e, M_IFNET); - return; + return (NULL); } if (V_ifindex_table != NULL) { memcpy((caddr_t)e, (caddr_t)V_ifindex_table, n/2); - free((caddr_t)V_ifindex_table, M_IFNET); + old = V_ifindex_table; } V_if_indexlim <<= 1; V_ifindex_table = e; + return (old); } /* @@ -500,11 +536,19 @@ if_alloc(u_char type) { struct ifnet *ifp; u_short idx; + void *old; ifp = malloc(sizeof(struct ifnet), M_IFNET, M_WAITOK|M_ZERO); + restart: IFNET_WLOCK(); - idx = ifindex_alloc(); - ifnet_setbyindex_locked(idx, IFNET_HOLD); + idx = ifindex_alloc(&old); + if (__predict_false(idx == USHRT_MAX)) { + IFNET_WUNLOCK(); + epoch_wait_preempt(net_epoch_preempt); + free(old, M_IFNET); + goto restart; + } + ifnet_setbyindex(idx, IFNET_HOLD); IFNET_WUNLOCK(); ifp->if_index = idx; ifp->if_type = type; @@ -525,9 +569,9 @@ if_alloc(u_char type) TASK_INIT(&ifp->if_linktask, 0, do_link_state_change, ifp); ifp->if_afdata_initialized = 0; IF_AFDATA_LOCK_INIT(ifp); - TAILQ_INIT(&ifp->if_addrhead); - TAILQ_INIT(&ifp->if_multiaddrs); - TAILQ_INIT(&ifp->if_groups); + CK_STAILQ_INIT(&ifp->if_addrhead); + CK_STAILQ_INIT(&ifp->if_multiaddrs); + CK_STAILQ_INIT(&ifp->if_groups); #ifdef MAC mac_ifnet_init(ifp); #endif @@ -573,6 +617,15 @@ if_free_internal(struct ifnet *ifp) free(ifp, M_IFNET); } +static void +if_destroy(epoch_context_t ctx) +{ + struct ifnet *ifp; + + ifp = __containerof(ctx, struct ifnet, if_epoch_ctx); + if_free_internal(ifp); +} + /* * Deregister an interface and free the associated storage. */ @@ -591,7 +644,7 @@ if_free(struct ifnet *ifp) IFNET_WUNLOCK(); if (refcount_release(&ifp->if_refcount)) - if_free_internal(ifp); + epoch_call(net_epoch_preempt, &ifp->if_epoch_ctx, if_destroy); CURVNET_RESTORE(); } @@ -614,7 +667,7 @@ if_rele(struct ifnet *ifp) if (!refcount_release(&ifp->if_refcount)) return; - if_free_internal(ifp); + epoch_call(net_epoch_preempt, &ifp->if_epoch_ctx, if_destroy); } void @@ -795,7 +848,7 @@ if_attach_internal(struct ifnet *ifp, int vmove, struct if_clone *ifc) sdl->sdl_len = masklen; while (namelen != 0) sdl->sdl_data[--namelen] = 0xff; - TAILQ_INSERT_HEAD(&ifp->if_addrhead, ifa, ifa_link); + CK_STAILQ_INSERT_HEAD(&ifp->if_addrhead, ifa, ifa_link); /* Reliably crash if used uninitialized. */ ifp->if_broadcastaddr = NULL; @@ -837,7 +890,7 @@ if_attach_internal(struct ifnet *ifp, int vmove, struct if_clone *ifc) * of the interface. */ for (ifa = ifp->if_addr; ifa != NULL; - ifa = TAILQ_NEXT(ifa, ifa_link)) { + ifa = CK_STAILQ_NEXT(ifa, ifa_link)) { if (ifa->ifa_addr->sa_family == AF_LINK) { sdl = (struct sockaddr_dl *)ifa->ifa_addr; sdl->sdl_index = ifp->if_index; @@ -847,7 +900,7 @@ if_attach_internal(struct ifnet *ifp, int vmove, struct if_clone *ifc) #endif IFNET_WLOCK(); - TAILQ_INSERT_TAIL(&V_ifnet, ifp, if_link); + CK_STAILQ_INSERT_TAIL(&V_ifnet, ifp, if_link); #ifdef VIMAGE curvnet->vnet_ifcnt++; #endif @@ -864,12 +917,24 @@ if_attach_internal(struct ifnet *ifp, int vmove, struct if_clone *ifc) rt_ifannouncemsg(ifp, IFAN_ARRIVAL); } +#ifndef __rtems__ +static void +if_epochalloc(void *dummy __unused) +{ + + net_epoch_preempt = epoch_alloc(EPOCH_PREEMPT); + net_epoch = epoch_alloc(0); +} +SYSINIT(ifepochalloc, SI_SUB_TASKQ + 1, SI_ORDER_ANY, + if_epochalloc, NULL); +#endif /* __rtems__ */ + static void if_attachdomain(void *dummy) { struct ifnet *ifp; - TAILQ_FOREACH(ifp, &V_ifnet, if_link) + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) if_attachdomain1(ifp); } SYSINIT(domainifattach, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_SECOND, @@ -911,8 +976,8 @@ if_purgeaddrs(struct ifnet *ifp) { struct ifaddr *ifa, *next; - /* XXX cannot hold IF_ADDR_WLOCK over called functions. */ - TAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, next) { + NET_EPOCH_ENTER(); + CK_STAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, next) { if (ifa->ifa_addr->sa_family == AF_LINK) continue; #ifdef INET @@ -937,10 +1002,11 @@ if_purgeaddrs(struct ifnet *ifp) } #endif /* INET6 */ IF_ADDR_WLOCK(ifp); - TAILQ_REMOVE(&ifp->if_addrhead, ifa, ifa_link); + CK_STAILQ_REMOVE(&ifp->if_addrhead, ifa, ifaddr, ifa_link); IF_ADDR_WUNLOCK(ifp); ifa_free(ifa); } + NET_EPOCH_EXIT(); } /* @@ -951,11 +1017,13 @@ static void if_purgemaddrs(struct ifnet *ifp) { struct ifmultiaddr *ifma; - struct ifmultiaddr *next; IF_ADDR_WLOCK(ifp); - TAILQ_FOREACH_SAFE(ifma, &ifp->if_multiaddrs, ifma_link, next) + while (!CK_STAILQ_EMPTY(&ifp->if_multiaddrs)) { + ifma = CK_STAILQ_FIRST(&ifp->if_multiaddrs); + CK_STAILQ_REMOVE(&ifp->if_multiaddrs, ifma, ifmultiaddr, ifma_link); if_delmulti_locked(ifp, ifma, 1); + } IF_ADDR_WUNLOCK(ifp); } @@ -1002,9 +1070,9 @@ if_detach_internal(struct ifnet *ifp, int vmove, struct if_clone **ifcp) ifp->if_vnet->vnet_state < SI_SUB_VNET_DONE) ? 1 : 0; #endif IFNET_WLOCK(); - TAILQ_FOREACH(iter, &V_ifnet, if_link) + CK_STAILQ_FOREACH(iter, &V_ifnet, if_link) if (iter == ifp) { - TAILQ_REMOVE(&V_ifnet, ifp, if_link); + CK_STAILQ_REMOVE(&V_ifnet, ifp, ifnet, if_link); found = 1; break; } @@ -1032,7 +1100,7 @@ if_detach_internal(struct ifnet *ifp, int vmove, struct if_clone **ifcp) #ifdef VIMAGE curvnet->vnet_ifcnt--; #endif - + epoch_wait_preempt(net_epoch_preempt); /* * In any case (destroy or vmove) detach us from the groups * and remove/wait for pending events on the taskq. @@ -1125,9 +1193,9 @@ if_detach_internal(struct ifnet *ifp, int vmove, struct if_clone **ifcp) /* We can now free link ifaddr. */ IF_ADDR_WLOCK(ifp); - if (!TAILQ_EMPTY(&ifp->if_addrhead)) { - ifa = TAILQ_FIRST(&ifp->if_addrhead); - TAILQ_REMOVE(&ifp->if_addrhead, ifa, ifa_link); + if (!CK_STAILQ_EMPTY(&ifp->if_addrhead)) { + ifa = CK_STAILQ_FIRST(&ifp->if_addrhead); + CK_STAILQ_REMOVE(&ifp->if_addrhead, ifa, ifaddr, ifa_link); IF_ADDR_WUNLOCK(ifp); ifa_free(ifa); } else @@ -1172,6 +1240,7 @@ if_vmove(struct ifnet *ifp, struct vnet *new_vnet) { struct if_clone *ifc; u_int bif_dlt, bif_hdrlen; + void *old; int rc; /* @@ -1212,10 +1281,16 @@ if_vmove(struct ifnet *ifp, struct vnet *new_vnet) * Switch to the context of the target vnet. */ CURVNET_SET_QUIET(new_vnet); - + restart: IFNET_WLOCK(); - ifp->if_index = ifindex_alloc(); - ifnet_setbyindex_locked(ifp->if_index, ifp); + ifp->if_index = ifindex_alloc(&old); + if (__predict_false(ifp->if_index == USHRT_MAX)) { + IFNET_WUNLOCK(); + epoch_wait_preempt(net_epoch_preempt); + free(old, M_IFNET); + goto restart; + } + ifnet_setbyindex(ifp->if_index, ifp); IFNET_WUNLOCK(); if_attach_internal(ifp, 1, ifc); @@ -1352,7 +1427,7 @@ if_addgroup(struct ifnet *ifp, const char *groupname) return (EINVAL); IFNET_WLOCK(); - TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) + CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) if (!strcmp(ifgl->ifgl_group->ifg_group, groupname)) { IFNET_WUNLOCK(); return (EEXIST); @@ -1371,7 +1446,7 @@ if_addgroup(struct ifnet *ifp, const char *groupname) return (ENOMEM); } - TAILQ_FOREACH(ifg, &V_ifg_head, ifg_next) + CK_STAILQ_FOREACH(ifg, &V_ifg_head, ifg_next) if (!strcmp(ifg->ifg_group, groupname)) break; @@ -1385,8 +1460,8 @@ if_addgroup(struct ifnet *ifp, const char *groupname) } strlcpy(ifg->ifg_group, groupname, sizeof(ifg->ifg_group)); ifg->ifg_refcnt = 0; - TAILQ_INIT(&ifg->ifg_members); - TAILQ_INSERT_TAIL(&V_ifg_head, ifg, ifg_next); + CK_STAILQ_INIT(&ifg->ifg_members); + CK_STAILQ_INSERT_TAIL(&V_ifg_head, ifg, ifg_next); new = 1; } @@ -1395,8 +1470,8 @@ if_addgroup(struct ifnet *ifp, const char *groupname) ifgm->ifgm_ifp = ifp; IF_ADDR_WLOCK(ifp); - TAILQ_INSERT_TAIL(&ifg->ifg_members, ifgm, ifgm_next); - TAILQ_INSERT_TAIL(&ifp->if_groups, ifgl, ifgl_next); + CK_STAILQ_INSERT_TAIL(&ifg->ifg_members, ifgm, ifgm_next); + CK_STAILQ_INSERT_TAIL(&ifp->if_groups, ifgl, ifgl_next); IF_ADDR_WUNLOCK(ifp); IFNET_WUNLOCK(); @@ -1416,9 +1491,10 @@ if_delgroup(struct ifnet *ifp, const char *groupname) { struct ifg_list *ifgl; struct ifg_member *ifgm; + int freeifgl; IFNET_WLOCK(); - TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) + CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) if (!strcmp(ifgl->ifgl_group->ifg_group, groupname)) break; if (ifgl == NULL) { @@ -1426,27 +1502,30 @@ if_delgroup(struct ifnet *ifp, const char *groupname) return (ENOENT); } + freeifgl = 0; IF_ADDR_WLOCK(ifp); - TAILQ_REMOVE(&ifp->if_groups, ifgl, ifgl_next); + CK_STAILQ_REMOVE(&ifp->if_groups, ifgl, ifg_list, ifgl_next); IF_ADDR_WUNLOCK(ifp); - TAILQ_FOREACH(ifgm, &ifgl->ifgl_group->ifg_members, ifgm_next) + CK_STAILQ_FOREACH(ifgm, &ifgl->ifgl_group->ifg_members, ifgm_next) if (ifgm->ifgm_ifp == ifp) break; - if (ifgm != NULL) { - TAILQ_REMOVE(&ifgl->ifgl_group->ifg_members, ifgm, ifgm_next); - free(ifgm, M_TEMP); - } + if (ifgm != NULL) + CK_STAILQ_REMOVE(&ifgl->ifgl_group->ifg_members, ifgm, ifg_member, ifgm_next); if (--ifgl->ifgl_group->ifg_refcnt == 0) { - TAILQ_REMOVE(&V_ifg_head, ifgl->ifgl_group, ifg_next); - IFNET_WUNLOCK(); + CK_STAILQ_REMOVE(&V_ifg_head, ifgl->ifgl_group, ifg_group, ifg_next); + freeifgl = 1; + } + IFNET_WUNLOCK(); + + epoch_wait_preempt(net_epoch_preempt); + if (freeifgl) { EVENTHANDLER_INVOKE(group_detach_event, ifgl->ifgl_group); free(ifgl->ifgl_group, M_TEMP); - } else - IFNET_WUNLOCK(); - + } + free(ifgm, M_TEMP); free(ifgl, M_TEMP); EVENTHANDLER_INVOKE(group_change_event, groupname); @@ -1463,38 +1542,39 @@ if_delgroups(struct ifnet *ifp) struct ifg_list *ifgl; struct ifg_member *ifgm; char groupname[IFNAMSIZ]; + int ifglfree; IFNET_WLOCK(); - while (!TAILQ_EMPTY(&ifp->if_groups)) { - ifgl = TAILQ_FIRST(&ifp->if_groups); + while (!CK_STAILQ_EMPTY(&ifp->if_groups)) { + ifgl = CK_STAILQ_FIRST(&ifp->if_groups); strlcpy(groupname, ifgl->ifgl_group->ifg_group, IFNAMSIZ); IF_ADDR_WLOCK(ifp); - TAILQ_REMOVE(&ifp->if_groups, ifgl, ifgl_next); + CK_STAILQ_REMOVE(&ifp->if_groups, ifgl, ifg_list, ifgl_next); IF_ADDR_WUNLOCK(ifp); - TAILQ_FOREACH(ifgm, &ifgl->ifgl_group->ifg_members, ifgm_next) + CK_STAILQ_FOREACH(ifgm, &ifgl->ifgl_group->ifg_members, ifgm_next) if (ifgm->ifgm_ifp == ifp) break; - if (ifgm != NULL) { - TAILQ_REMOVE(&ifgl->ifgl_group->ifg_members, ifgm, + if (ifgm != NULL) + CK_STAILQ_REMOVE(&ifgl->ifgl_group->ifg_members, ifgm, ifg_member, ifgm_next); - free(ifgm, M_TEMP); + ifglfree = 0; + if (--ifgl->ifgl_group->ifg_refcnt == 0) { + CK_STAILQ_REMOVE(&V_ifg_head, ifgl->ifgl_group, ifg_group, ifg_next); + ifglfree = 1; } - if (--ifgl->ifgl_group->ifg_refcnt == 0) { - TAILQ_REMOVE(&V_ifg_head, ifgl->ifgl_group, ifg_next); - IFNET_WUNLOCK(); + IFNET_WUNLOCK(); + epoch_wait_preempt(net_epoch_preempt); + free(ifgm, M_TEMP); + if (ifglfree) { EVENTHANDLER_INVOKE(group_detach_event, - ifgl->ifgl_group); + ifgl->ifgl_group); free(ifgl->ifgl_group, M_TEMP); - } else - IFNET_WUNLOCK(); - - free(ifgl, M_TEMP); - + } EVENTHANDLER_INVOKE(group_change_event, groupname); IFNET_WLOCK(); @@ -1502,31 +1582,56 @@ if_delgroups(struct ifnet *ifp) IFNET_WUNLOCK(); } +static char * +ifgr_group_get(void *ifgrp) +{ + union ifgroupreq_union *ifgrup; + + ifgrup = ifgrp; +#ifdef COMPAT_FREEBSD32 + if (SV_CURPROC_FLAG(SV_ILP32)) + return (&ifgrup->ifgr32.ifgr_ifgru.ifgru_group[0]); +#endif + return (&ifgrup->ifgr.ifgr_ifgru.ifgru_group[0]); +} + +static struct ifg_req * +ifgr_groups_get(void *ifgrp) +{ + union ifgroupreq_union *ifgrup; + + ifgrup = ifgrp; +#ifdef COMPAT_FREEBSD32 + if (SV_CURPROC_FLAG(SV_ILP32)) + return ((struct ifg_req *)(uintptr_t) + ifgrup->ifgr32.ifgr_ifgru.ifgru_groups); +#endif + return (ifgrup->ifgr.ifgr_ifgru.ifgru_groups); +} + /* - * Stores all groups from an interface in memory pointed - * to by data + * Stores all groups from an interface in memory pointed to by ifgr. */ static int -if_getgroup(struct ifgroupreq *data, struct ifnet *ifp) +if_getgroup(struct ifgroupreq *ifgr, struct ifnet *ifp) { int len, error; struct ifg_list *ifgl; struct ifg_req ifgrq, *ifgp; - struct ifgroupreq *ifgr = data; if (ifgr->ifgr_len == 0) { IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) + CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) ifgr->ifgr_len += sizeof(struct ifg_req); IF_ADDR_RUNLOCK(ifp); return (0); } len = ifgr->ifgr_len; - ifgp = ifgr->ifgr_groups; + ifgp = ifgr_groups_get(ifgr); /* XXX: wire */ IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) { + CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) { if (len < sizeof(ifgrq)) { IF_ADDR_RUNLOCK(ifp); return (EINVAL); @@ -1547,19 +1652,18 @@ if_getgroup(struct ifgroupreq *data, struct ifnet *ifp) } /* - * Stores all members of a group in memory pointed to by data + * Stores all members of a group in memory pointed to by igfr */ static int -if_getgroupmembers(struct ifgroupreq *data) +if_getgroupmembers(struct ifgroupreq *ifgr) { - struct ifgroupreq *ifgr = data; struct ifg_group *ifg; struct ifg_member *ifgm; struct ifg_req ifgrq, *ifgp; int len, error; IFNET_RLOCK(); - TAILQ_FOREACH(ifg, &V_ifg_head, ifg_next) + CK_STAILQ_FOREACH(ifg, &V_ifg_head, ifg_next) if (!strcmp(ifg->ifg_group, ifgr->ifgr_name)) break; if (ifg == NULL) { @@ -1568,15 +1672,15 @@ if_getgroupmembers(struct ifgroupreq *data) } if (ifgr->ifgr_len == 0) { - TAILQ_FOREACH(ifgm, &ifg->ifg_members, ifgm_next) + CK_STAILQ_FOREACH(ifgm, &ifg->ifg_members, ifgm_next) ifgr->ifgr_len += sizeof(ifgrq); IFNET_RUNLOCK(); return (0); } len = ifgr->ifgr_len; - ifgp = ifgr->ifgr_groups; - TAILQ_FOREACH(ifgm, &ifg->ifg_members, ifgm_next) { + ifgp = ifgr_groups_get(ifgr); + CK_STAILQ_FOREACH(ifgm, &ifg->ifg_members, ifgm_next) { if (len < sizeof(ifgrq)) { IFNET_RUNLOCK(); return (EINVAL); @@ -1736,19 +1840,28 @@ ifa_ref(struct ifaddr *ifa) refcount_acquire(&ifa->ifa_refcnt); } +static void +ifa_destroy(epoch_context_t ctx) +{ + struct ifaddr *ifa; + + ifa = __containerof(ctx, struct ifaddr, ifa_epoch_ctx); + counter_u64_free(ifa->ifa_opackets); + counter_u64_free(ifa->ifa_ipackets); + counter_u64_free(ifa->ifa_obytes); + counter_u64_free(ifa->ifa_ibytes); + free(ifa, M_IFADDR); +} + void ifa_free(struct ifaddr *ifa) { - if (refcount_release(&ifa->ifa_refcnt)) { - counter_u64_free(ifa->ifa_opackets); - counter_u64_free(ifa->ifa_ipackets); - counter_u64_free(ifa->ifa_obytes); - counter_u64_free(ifa->ifa_ibytes); - free(ifa, M_IFADDR); - } + if (refcount_release(&ifa->ifa_refcnt)) + epoch_call(net_epoch_preempt, &ifa->ifa_epoch_ctx, ifa_destroy); } + static int ifa_maintain_loopback_route(int cmd, const char *otype, struct ifaddr *ifa, struct sockaddr *ia) @@ -1770,9 +1883,10 @@ ifa_maintain_loopback_route(int cmd, const char *otype, struct ifaddr *ifa, error = rtrequest1_fib(cmd, &info, NULL, ifp->if_fib); - if (error != 0) - log(LOG_DEBUG, "%s: %s failed for interface %s: %u\n", - __func__, otype, if_name(ifp), error); + if (error != 0 && + !(cmd == RTM_ADD && error == EEXIST) && + !(cmd == RTM_DELETE && error == ENOENT)) + if_printf(ifp, "%s failed: %d\n", otype, error); return (error); } @@ -1815,22 +1929,18 @@ ifa_switch_loopback_route(struct ifaddr *ifa, struct sockaddr *ia) * Locate an interface based on a complete address. */ /*ARGSUSED*/ -static struct ifaddr * -ifa_ifwithaddr_internal(const struct sockaddr *addr, int getref) +struct ifaddr * +ifa_ifwithaddr(const struct sockaddr *addr) { struct ifnet *ifp; struct ifaddr *ifa; - IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { - IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + MPASS(in_epoch()); + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != addr->sa_family) continue; if (sa_equal(addr, ifa->ifa_addr)) { - if (getref) - ifa_ref(ifa); - IF_ADDR_RUNLOCK(ifp); goto done; } /* IP6 doesn't have broadcast */ @@ -1838,32 +1948,24 @@ ifa_ifwithaddr_internal(const struct sockaddr *addr, int getref) ifa->ifa_broadaddr && ifa->ifa_broadaddr->sa_len != 0 && sa_equal(ifa->ifa_broadaddr, addr)) { - if (getref) - ifa_ref(ifa); - IF_ADDR_RUNLOCK(ifp); goto done; } } - IF_ADDR_RUNLOCK(ifp); } ifa = NULL; done: - IFNET_RUNLOCK_NOSLEEP(); return (ifa); } -struct ifaddr * -ifa_ifwithaddr(const struct sockaddr *addr) -{ - - return (ifa_ifwithaddr_internal(addr, 1)); -} - int ifa_ifwithaddr_check(const struct sockaddr *addr) { + int rc; - return (ifa_ifwithaddr_internal(addr, 0) != NULL); + NET_EPOCH_ENTER(); + rc = (ifa_ifwithaddr(addr) != NULL); + NET_EPOCH_EXIT(); + return (rc); } /* @@ -1876,28 +1978,23 @@ ifa_ifwithbroadaddr(const struct sockaddr *addr, int fibnum) struct ifnet *ifp; struct ifaddr *ifa; - IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + MPASS(in_epoch()); + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if ((fibnum != RT_ALL_FIBS) && (ifp->if_fib != fibnum)) continue; - IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != addr->sa_family) continue; if ((ifp->if_flags & IFF_BROADCAST) && ifa->ifa_broadaddr && ifa->ifa_broadaddr->sa_len != 0 && sa_equal(ifa->ifa_broadaddr, addr)) { - ifa_ref(ifa); - IF_ADDR_RUNLOCK(ifp); goto done; } } - IF_ADDR_RUNLOCK(ifp); } ifa = NULL; done: - IFNET_RUNLOCK_NOSLEEP(); return (ifa); } @@ -1911,28 +2008,23 @@ ifa_ifwithdstaddr(const struct sockaddr *addr, int fibnum) struct ifnet *ifp; struct ifaddr *ifa; - IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + MPASS(in_epoch()); + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if ((ifp->if_flags & IFF_POINTOPOINT) == 0) continue; if ((fibnum != RT_ALL_FIBS) && (ifp->if_fib != fibnum)) continue; - IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != addr->sa_family) continue; if (ifa->ifa_dstaddr != NULL && sa_equal(addr, ifa->ifa_dstaddr)) { - ifa_ref(ifa); - IF_ADDR_RUNLOCK(ifp); goto done; } } - IF_ADDR_RUNLOCK(ifp); } ifa = NULL; done: - IFNET_RUNLOCK_NOSLEEP(); return (ifa); } @@ -1949,6 +2041,7 @@ ifa_ifwithnet(const struct sockaddr *addr, int ignore_ptp, int fibnum) u_int af = addr->sa_family; const char *addr_data = addr->sa_data, *cplim; + MPASS(in_epoch()); /* * AF_LINK addresses can be looked up directly by their index number, * so do that if we can. @@ -1965,12 +2058,10 @@ ifa_ifwithnet(const struct sockaddr *addr, int ignore_ptp, int fibnum) * on ifa_maybe once we find one, as we release the IF_ADDR_RLOCK() that * kept it stable when we move onto the next interface. */ - IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if ((fibnum != RT_ALL_FIBS) && (ifp->if_fib != fibnum)) continue; - IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { const char *cp, *cp2, *cp3; if (ifa->ifa_addr->sa_family != af) @@ -1987,7 +2078,6 @@ next: continue; */ if (ifa->ifa_dstaddr != NULL && sa_equal(addr, ifa->ifa_dstaddr)) { - ifa_ref(ifa); IF_ADDR_RUNLOCK(ifp); goto done; } @@ -2021,21 +2111,14 @@ next: continue; ifa_preferred(ifa_maybe, ifa) || rn_refines((caddr_t)ifa->ifa_netmask, (caddr_t)ifa_maybe->ifa_netmask)) { - if (ifa_maybe != NULL) - ifa_free(ifa_maybe); ifa_maybe = ifa; - ifa_ref(ifa_maybe); } } } - IF_ADDR_RUNLOCK(ifp); } ifa = ifa_maybe; ifa_maybe = NULL; done: - IFNET_RUNLOCK_NOSLEEP(); - if (ifa_maybe != NULL) - ifa_free(ifa_maybe); return (ifa); } @@ -2054,8 +2137,8 @@ ifaof_ifpforaddr(const struct sockaddr *addr, struct ifnet *ifp) if (af >= AF_MAX) return (NULL); - IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + MPASS(in_epoch()); + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != af) continue; if (ifa_maybe == NULL) @@ -2084,9 +2167,6 @@ ifaof_ifpforaddr(const struct sockaddr *addr, struct ifnet *ifp) } ifa = ifa_maybe; done: - if (ifa != NULL) - ifa_ref(ifa); - IF_ADDR_RUNLOCK(ifp); return (ifa); } @@ -2122,14 +2202,19 @@ link_rtrequest(int cmd, struct rtentry *rt, struct rt_addrinfo *info) if (cmd != RTM_ADD || ((ifa = rt->rt_ifa) == NULL) || ((ifp = ifa->ifa_ifp) == NULL) || ((dst = rt_key(rt)) == NULL)) return; + NET_EPOCH_ENTER(); ifa = ifaof_ifpforaddr(dst, ifp); if (ifa) { oifa = rt->rt_ifa; + if (oifa != ifa) { + ifa_free(oifa); + ifa_ref(ifa); + } rt->rt_ifa = ifa; - ifa_free(oifa); if (ifa->ifa_rtrequest && ifa->ifa_rtrequest != link_rtrequest) ifa->ifa_rtrequest(cmd, rt, info); } + NET_EPOCH_EXIT(); } struct sockaddr_dl * @@ -2177,7 +2262,7 @@ if_unroute(struct ifnet *ifp, int flag, int fam) ifp->if_flags &= ~flag; getmicrotime(&ifp->if_lastchange); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) if (fam == PF_UNSPEC || (fam == ifa->ifa_addr->sa_family)) pfctlinput(PRC_IFDOWN, ifa->ifa_addr); ifp->if_qflush(ifp); @@ -2200,7 +2285,7 @@ if_route(struct ifnet *ifp, int flag, int fam) ifp->if_flags |= flag; getmicrotime(&ifp->if_lastchange); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) if (fam == PF_UNSPEC || (fam == ifa->ifa_addr->sa_family)) pfctlinput(PRC_IFUP, ifa->ifa_addr); if (ifp->if_carp) @@ -2255,7 +2340,7 @@ do_link_state_change(void *arg, int pending) if (ifp->if_carp) (*carp_linkstate_p)(ifp); if (ifp->if_bridge) - (*bridge_linkstate_p)(ifp); + ifp->if_bridge_linkstate(ifp); if (ifp->if_lagg) (*lagg_linkstate_p)(ifp, link_state); @@ -2266,7 +2351,7 @@ do_link_state_change(void *arg, int pending) if (pending > 1) if_printf(ifp, "%d link states coalesced\n", pending); if (log_link_state_change) - log(LOG_NOTICE, "%s: link state changed to %s\n", ifp->if_xname, + if_printf(ifp, "link state changed to %s\n", (link_state == LINK_STATE_UP) ? "UP" : "DOWN" ); EVENTHANDLER_INVOKE(ifnet_link_event, ifp, link_state); CURVNET_RESTORE(); @@ -2332,7 +2417,7 @@ ifunit_ref(const char *name) struct ifnet *ifp; IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (strncmp(name, ifp->if_xname, IFNAMSIZ) == 0 && !(ifp->if_flags & IFF_DYING)) break; @@ -2349,7 +2434,7 @@ ifunit(const char *name) struct ifnet *ifp; IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (strncmp(name, ifp->if_xname, IFNAMSIZ) == 0) break; } @@ -2358,13 +2443,13 @@ ifunit(const char *name) } static void * -ifr_buffer_get_buffer(struct thread *td, void *data) +ifr_buffer_get_buffer(void *data) { union ifreq_union *ifrup; ifrup = data; #ifdef COMPAT_FREEBSD32 - if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) + if (SV_CURPROC_FLAG(SV_ILP32)) return ((void *)(uintptr_t) ifrup->ifr32.ifr_ifru.ifru_buffer.buffer); #endif @@ -2372,13 +2457,13 @@ ifr_buffer_get_buffer(struct thread *td, void *data) } static void -ifr_buffer_set_buffer_null(struct thread *td, void *data) +ifr_buffer_set_buffer_null(void *data) { union ifreq_union *ifrup; ifrup = data; #ifdef COMPAT_FREEBSD32 - if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) + if (SV_CURPROC_FLAG(SV_ILP32)) ifrup->ifr32.ifr_ifru.ifru_buffer.buffer = 0; else #endif @@ -2386,26 +2471,26 @@ ifr_buffer_set_buffer_null(struct thread *td, void *data) } static size_t -ifr_buffer_get_length(struct thread *td, void *data) +ifr_buffer_get_length(void *data) { union ifreq_union *ifrup; ifrup = data; #ifdef COMPAT_FREEBSD32 - if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) + if (SV_CURPROC_FLAG(SV_ILP32)) return (ifrup->ifr32.ifr_ifru.ifru_buffer.length); #endif return (ifrup->ifr.ifr_ifru.ifru_buffer.length); } static void -ifr_buffer_set_length(struct thread *td, void *data, size_t len) +ifr_buffer_set_length(void *data, size_t len) { union ifreq_union *ifrup; ifrup = data; #ifdef COMPAT_FREEBSD32 - if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) + if (SV_CURPROC_FLAG(SV_ILP32)) ifrup->ifr32.ifr_ifru.ifru_buffer.length = len; else #endif @@ -2486,12 +2571,12 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, struct thread *td) else { /* space for terminating nul */ descrlen = strlen(ifp->if_description) + 1; - if (ifr_buffer_get_length(td, ifr) < descrlen) - ifr_buffer_set_buffer_null(td, ifr); + if (ifr_buffer_get_length(ifr) < descrlen) + ifr_buffer_set_buffer_null(ifr); else error = copyout(ifp->if_description, - ifr_buffer_get_buffer(td, ifr), descrlen); - ifr_buffer_set_length(td, ifr, descrlen); + ifr_buffer_get_buffer(ifr), descrlen); + ifr_buffer_set_length(ifr, descrlen); } sx_sunlock(&ifdescr_sx); break; @@ -2507,15 +2592,15 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, struct thread *td) * length parameter is supposed to count the * terminating nul in. */ - if (ifr_buffer_get_length(td, ifr) > ifdescr_maxlen) + if (ifr_buffer_get_length(ifr) > ifdescr_maxlen) return (ENAMETOOLONG); - else if (ifr_buffer_get_length(td, ifr) == 0) + else if (ifr_buffer_get_length(ifr) == 0) descrbuf = NULL; else { - descrbuf = malloc(ifr_buffer_get_length(td, ifr), + descrbuf = malloc(ifr_buffer_get_length(ifr), M_IFDESCR, M_WAITOK | M_ZERO); - error = copyin(ifr_buffer_get_buffer(td, ifr), descrbuf, - ifr_buffer_get_length(td, ifr) - 1); + error = copyin(ifr_buffer_get_buffer(ifr), descrbuf, + ifr_buffer_get_length(ifr) - 1); if (error) { free(descrbuf, M_IFDESCR); break; @@ -2569,8 +2654,7 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, struct thread *td) else if (ifp->if_pcount == 0) ifp->if_flags &= ~IFF_PROMISC; if (log_promisc_mode_change) - log(LOG_INFO, "%s: permanently promiscuous mode %s\n", - ifp->if_xname, + if_printf(ifp, "permanently promiscuous mode %s\n", ((new_flags & IFF_PPROMISC) ? "enabled" : "disabled")); } @@ -2633,8 +2717,7 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, struct thread *td) rt_ifannouncemsg(ifp, IFAN_DEPARTURE); EVENTHANDLER_INVOKE(ifnet_departure_event, ifp); - log(LOG_INFO, "%s: changing name to '%s'\n", - ifp->if_xname, new_name); + if_printf(ifp, "changing name to '%s'\n", new_name); IF_ADDR_WLOCK(ifp); strlcpy(ifp->if_xname, new_name, sizeof(ifp->if_xname)); @@ -2709,6 +2792,9 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, struct thread *td) if (error == 0) { getmicrotime(&ifp->if_lastchange); rt_ifmsg(ifp); +#ifdef INET + NETDUMP_REINIT(ifp); +#endif } /* * If the link MTU changed, do network layer specific procedure. @@ -2805,34 +2891,28 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, struct thread *td) error = if_gethwaddr(ifp, ifr); break; - case SIOCAIFGROUP: - { - struct ifgroupreq *ifgr = (struct ifgroupreq *)ifr; - + CASE_IOC_IFGROUPREQ(SIOCAIFGROUP): error = priv_check(td, PRIV_NET_ADDIFGROUP); if (error) return (error); - if ((error = if_addgroup(ifp, ifgr->ifgr_group))) + if ((error = if_addgroup(ifp, + ifgr_group_get((struct ifgroupreq *)data)))) return (error); break; - } - case SIOCGIFGROUP: - if ((error = if_getgroup((struct ifgroupreq *)ifr, ifp))) + CASE_IOC_IFGROUPREQ(SIOCGIFGROUP): + if ((error = if_getgroup((struct ifgroupreq *)data, ifp))) return (error); break; - case SIOCDIFGROUP: - { - struct ifgroupreq *ifgr = (struct ifgroupreq *)ifr; - + CASE_IOC_IFGROUPREQ(SIOCDIFGROUP): error = priv_check(td, PRIV_NET_DELIFGROUP); if (error) return (error); - if ((error = if_delgroup(ifp, ifgr->ifgr_group))) + if ((error = if_delgroup(ifp, + ifgr_group_get((struct ifgroupreq *)data)))) return (error); break; - } #ifdef __rtems__ case RTEMS_SIOSIFINPUT: @@ -2868,12 +2948,48 @@ struct ifconf32 { #define SIOCGIFCONF32 _IOWR('i', 36, struct ifconf32) #endif +#ifdef COMPAT_FREEBSD32 +static void +ifmr_init(struct ifmediareq *ifmr, caddr_t data) +{ + struct ifmediareq32 *ifmr32; + + ifmr32 = (struct ifmediareq32 *)data; + memcpy(ifmr->ifm_name, ifmr32->ifm_name, + sizeof(ifmr->ifm_name)); + ifmr->ifm_current = ifmr32->ifm_current; + ifmr->ifm_mask = ifmr32->ifm_mask; + ifmr->ifm_status = ifmr32->ifm_status; + ifmr->ifm_active = ifmr32->ifm_active; + ifmr->ifm_count = ifmr32->ifm_count; + ifmr->ifm_ulist = (int *)(uintptr_t)ifmr32->ifm_ulist; +} + +static void +ifmr_update(const struct ifmediareq *ifmr, caddr_t data) +{ + struct ifmediareq32 *ifmr32; + + ifmr32 = (struct ifmediareq32 *)data; + ifmr32->ifm_current = ifmr->ifm_current; + ifmr32->ifm_mask = ifmr->ifm_mask; + ifmr32->ifm_status = ifmr->ifm_status; + ifmr32->ifm_active = ifmr->ifm_active; + ifmr32->ifm_count = ifmr->ifm_count; +} +#endif + /* * Interface ioctls. */ int ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td) { +#ifdef COMPAT_FREEBSD32 + caddr_t saved_data = NULL; + struct ifmediareq ifmr; +#endif + struct ifmediareq *ifmrp; struct ifnet *ifp; struct ifreq *ifr; int error; @@ -2918,8 +3034,21 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td) } #endif } - ifr = (struct ifreq *)data; + ifmrp = NULL; +#ifdef COMPAT_FREEBSD32 + switch (cmd) { + case SIOCGIFMEDIA32: + case SIOCGIFXMEDIA32: + ifmrp = &ifmr; + ifmr_init(ifmrp, data); + cmd = _IOC_NEWTYPE(cmd, struct ifmediareq); + saved_data = data; + data = (caddr_t)ifmrp; + } +#endif + + ifr = (struct ifreq *)data; switch (cmd) { #ifdef VIMAGE case SIOCSIFRVNET: @@ -2927,8 +3056,7 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td) if (error == 0) error = if_vmove_reclaim(td, ifr->ifr_name, ifr->ifr_jid); - CURVNET_RESTORE(); - return (error); + goto out_noref; #endif case SIOCIFCREATE: case SIOCIFCREATE2: @@ -2937,23 +3065,21 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td) error = if_clone_create(ifr->ifr_name, sizeof(ifr->ifr_name), cmd == SIOCIFCREATE2 ? ifr_data_get_ptr(ifr) : NULL); - CURVNET_RESTORE(); - return (error); + goto out_noref; case SIOCIFDESTROY: error = priv_check(td, PRIV_NET_IFDESTROY); if (error == 0) error = if_clone_destroy(ifr->ifr_name); - CURVNET_RESTORE(); - return (error); + goto out_noref; case SIOCIFGCLONERS: error = if_clone_list((struct if_clonereq *)data); - CURVNET_RESTORE(); - return (error); - case SIOCGIFGMEMB: + goto out_noref; + + CASE_IOC_IFGROUPREQ(SIOCGIFGMEMB): error = if_getgroupmembers((struct ifgroupreq *)data); - CURVNET_RESTORE(); - return (error); + goto out_noref; + #if defined(INET) || defined(INET6) case SIOCSVH: case SIOCGVH: @@ -2961,29 +3087,24 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td) error = EPROTONOSUPPORT; else error = (*carp_ioctl_p)(ifr, cmd, td); - CURVNET_RESTORE(); - return (error); + goto out_noref; #endif } ifp = ifunit_ref(ifr->ifr_name); if (ifp == NULL) { - CURVNET_RESTORE(); - return (ENXIO); + error = ENXIO; + goto out_noref; } error = ifhwioctl(cmd, ifp, data, td); - if (error != ENOIOCTL) { - if_rele(ifp); - CURVNET_RESTORE(); - return (error); - } + if (error != ENOIOCTL) + goto out_ref; oif_flags = ifp->if_flags; if (so->so_proto == NULL) { - if_rele(ifp); - CURVNET_RESTORE(); - return (EOPNOTSUPP); + error = EOPNOTSUPP; + goto out_ref; } /* @@ -3008,7 +3129,19 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td) in6_if_up(ifp); #endif } + +out_ref: if_rele(ifp); +out_noref: +#ifdef COMPAT_FREEBSD32 + if (ifmrp != NULL) { + KASSERT((cmd == SIOCGIFMEDIA || cmd == SIOCGIFXMEDIA), + ("ifmrp non-NULL, but cmd is not an ifmedia req 0x%lx", + cmd)); + data = saved_data; + ifmr_update(ifmrp, data); + } +#endif CURVNET_RESTORE(); return (error); } @@ -3103,8 +3236,7 @@ ifpromisc(struct ifnet *ifp, int pswitch) /* If promiscuous mode status has changed, log a message */ if (error == 0 && ((ifp->if_flags ^ oldflags) & IFF_PROMISC) && log_promisc_mode_change) - log(LOG_INFO, "%s: promiscuous mode %s\n", - ifp->if_xname, + if_printf(ifp, "promiscuous mode %s\n", (ifp->if_flags & IFF_PROMISC) ? "enabled" : "disabled"); return (error); } @@ -3143,14 +3275,14 @@ again: valid_len = 0; IFNET_RLOCK(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { int addrs; /* - * Zero the ifr_name buffer to make sure we don't - * disclose the contents of the stack. + * Zero the ifr to make sure we don't disclose the contents + * of the stack. */ - memset(ifr.ifr_name, 0, sizeof(ifr.ifr_name)); + memset(&ifr, 0, sizeof(ifr)); if (strlcpy(ifr.ifr_name, ifp->if_xname, sizeof(ifr.ifr_name)) >= sizeof(ifr.ifr_name)) { @@ -3161,14 +3293,20 @@ again: addrs = 0; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { struct sockaddr *sa = ifa->ifa_addr; if (prison_if(curthread->td_ucred, sa) != 0) continue; addrs++; if (sa->sa_len <= sizeof(*sa)) { - ifr.ifr_addr = *sa; + if (sa->sa_len < sizeof(*sa)) { + memset(&ifr.ifr_ifru.ifru_addr, 0, + sizeof(ifr.ifr_ifru.ifru_addr)); + memcpy(&ifr.ifr_ifru.ifru_addr, sa, + sa->sa_len); + } else + ifr.ifr_ifru.ifru_addr = *sa; sbuf_bcat(sb, &ifr, sizeof(ifr)); max_len += sizeof(ifr); } else { @@ -3184,7 +3322,6 @@ again: } IF_ADDR_RUNLOCK(ifp); if (addrs == 0) { - bzero((caddr_t)&ifr.ifr_addr, sizeof(ifr.ifr_addr)); sbuf_bcat(sb, &ifr, sizeof(ifr)); max_len += sizeof(ifr); @@ -3228,7 +3365,7 @@ if_findmulti(struct ifnet *ifp, const struct sockaddr *sa) IF_ADDR_LOCK_ASSERT(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (sa->sa_family == AF_LINK) { if (sa_dl_equal(ifma->ifma_addr, sa)) break; @@ -3295,8 +3432,11 @@ if_allocmulti(struct ifnet *ifp, struct sockaddr *sa, struct sockaddr *llsa, * counting, notifying the driver, handling routing messages, and releasing * any dependent link layer state. */ +#ifdef MCAST_VERBOSE +extern void kdb_backtrace(void); +#endif static void -if_freemulti(struct ifmultiaddr *ifma) +if_freemulti_internal(struct ifmultiaddr *ifma) { KASSERT(ifma->ifma_refcount == 0, ("if_freemulti: refcount %d", @@ -3304,10 +3444,33 @@ if_freemulti(struct ifmultiaddr *ifma) if (ifma->ifma_lladdr != NULL) free(ifma->ifma_lladdr, M_IFMADDR); +#ifdef MCAST_VERBOSE + kdb_backtrace(); + printf("%s freeing ifma: %p\n", __func__, ifma); +#endif free(ifma->ifma_addr, M_IFMADDR); free(ifma, M_IFMADDR); } +static void +if_destroymulti(epoch_context_t ctx) +{ + struct ifmultiaddr *ifma; + + ifma = __containerof(ctx, struct ifmultiaddr, ifma_epoch_ctx); + if_freemulti_internal(ifma); +} + +void +if_freemulti(struct ifmultiaddr *ifma) +{ + KASSERT(ifma->ifma_refcount == 0, ("if_freemulti_epoch: refcount %d", + ifma->ifma_refcount)); + + epoch_call(net_epoch_preempt, &ifma->ifma_epoch_ctx, if_destroymulti); +} + + /* * Register an additional multicast address with a network interface. * @@ -3335,6 +3498,12 @@ if_addmulti(struct ifnet *ifp, struct sockaddr *sa, struct sockaddr_dl sdl; int error; +#ifdef INET + IN_MULTI_LIST_UNLOCK_ASSERT(); +#endif +#ifdef INET6 + IN6_MULTI_LIST_UNLOCK_ASSERT(); +#endif /* * If the address is already present, return a new reference to it; * otherwise, allocate storage and set up a new address. @@ -3395,7 +3564,7 @@ if_addmulti(struct ifnet *ifp, struct sockaddr *sa, error = ENOMEM; goto free_llsa_out; } - TAILQ_INSERT_HEAD(&ifp->if_multiaddrs, ll_ifma, + CK_STAILQ_INSERT_HEAD(&ifp->if_multiaddrs, ll_ifma, ifma_link); } else ll_ifma->ifma_refcount++; @@ -3407,7 +3576,7 @@ if_addmulti(struct ifnet *ifp, struct sockaddr *sa, * referenced link layer address. Add the primary address to the * ifnet address list. */ - TAILQ_INSERT_HEAD(&ifp->if_multiaddrs, ifma, ifma_link); + CK_STAILQ_INSERT_HEAD(&ifp->if_multiaddrs, ifma, ifma_link); if (retifma != NULL) *retifma = ifma; @@ -3459,7 +3628,7 @@ if_delmulti(struct ifnet *ifp, struct sockaddr *sa) struct ifnet *oifp; IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(oifp, &V_ifnet, if_link) + CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link) if (ifp == oifp) break; if (ifp != oifp) @@ -3499,11 +3668,17 @@ if_delallmulti(struct ifnet *ifp) struct ifmultiaddr *next; IF_ADDR_WLOCK(ifp); - TAILQ_FOREACH_SAFE(ifma, &ifp->if_multiaddrs, ifma_link, next) + CK_STAILQ_FOREACH_SAFE(ifma, &ifp->if_multiaddrs, ifma_link, next) if_delmulti_locked(ifp, ifma, 0); IF_ADDR_WUNLOCK(ifp); } +void +if_delmulti_ifma(struct ifmultiaddr *ifma) +{ + if_delmulti_ifma_flags(ifma, 0); +} + /* * Delete a multicast group membership by group membership pointer. * Network-layer protocol domains must use this routine. @@ -3511,11 +3686,14 @@ if_delallmulti(struct ifnet *ifp) * It is safe to call this routine if the ifp disappeared. */ void -if_delmulti_ifma(struct ifmultiaddr *ifma) +if_delmulti_ifma_flags(struct ifmultiaddr *ifma, int flags) { struct ifnet *ifp; int lastref; - + MCDPRINTF("%s freeing ifma: %p\n", __func__, ifma); +#ifdef INET + IN_MULTI_LIST_UNLOCK_ASSERT(); +#endif ifp = ifma->ifma_ifp; #ifdef DIAGNOSTIC if (ifp == NULL) { @@ -3524,13 +3702,11 @@ if_delmulti_ifma(struct ifmultiaddr *ifma) struct ifnet *oifp; IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(oifp, &V_ifnet, if_link) + CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link) if (ifp == oifp) break; - if (ifp != oifp) { - printf("%s: ifnet %p disappeared\n", __func__, ifp); + if (ifp != oifp) ifp = NULL; - } IFNET_RUNLOCK_NOSLEEP(); } #endif @@ -3540,7 +3716,7 @@ if_delmulti_ifma(struct ifmultiaddr *ifma) if (ifp != NULL) IF_ADDR_WLOCK(ifp); - lastref = if_delmulti_locked(ifp, ifma, 0); + lastref = if_delmulti_locked(ifp, ifma, flags); if (ifp != NULL) { /* @@ -3574,6 +3750,7 @@ if_delmulti_locked(struct ifnet *ifp, struct ifmultiaddr *ifma, int detaching) } ifp = ifma->ifma_ifp; + MCDPRINTF("%s freeing %p from %s \n", __func__, ifma, ifp ? ifp->if_xname : ""); /* * If the ifnet is detaching, null out references to ifnet, @@ -3599,6 +3776,9 @@ if_delmulti_locked(struct ifnet *ifp, struct ifmultiaddr *ifma, int detaching) if (--ifma->ifma_refcount > 0) return 0; + if (ifp != NULL && detaching == 0) + CK_STAILQ_REMOVE(&ifp->if_multiaddrs, ifma, ifmultiaddr, ifma_link); + /* * If this ifma is a network-layer ifma, a link-layer ifma may * have been associated with it. Release it first if so. @@ -3611,18 +3791,21 @@ if_delmulti_locked(struct ifnet *ifp, struct ifmultiaddr *ifma, int detaching) ll_ifma->ifma_ifp = NULL; /* XXX */ if (--ll_ifma->ifma_refcount == 0) { if (ifp != NULL) { - TAILQ_REMOVE(&ifp->if_multiaddrs, ll_ifma, + CK_STAILQ_REMOVE(&ifp->if_multiaddrs, ll_ifma, ifmultiaddr, ifma_link); } if_freemulti(ll_ifma); } } +#ifdef INVARIANTS + if (ifp) { + struct ifmultiaddr *ifmatmp; - if (ifp != NULL) - TAILQ_REMOVE(&ifp->if_multiaddrs, ifma, ifma_link); - + CK_STAILQ_FOREACH(ifmatmp, &ifp->if_multiaddrs, ifma_link) + MPASS(ifma != ifmatmp); + } +#endif if_freemulti(ifma); - /* * The last reference to this instance of struct ifmultiaddr * was released; the hardware should be notified of this change. @@ -3644,38 +3827,36 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len) struct sockaddr_dl *sdl; struct ifaddr *ifa; struct ifreq ifr; + int rc; - IF_ADDR_RLOCK(ifp); + rc = 0; + NET_EPOCH_ENTER(); ifa = ifp->if_addr; if (ifa == NULL) { - IF_ADDR_RUNLOCK(ifp); - return (EINVAL); + rc = EINVAL; + goto out; } - ifa_ref(ifa); - IF_ADDR_RUNLOCK(ifp); + sdl = (struct sockaddr_dl *)ifa->ifa_addr; if (sdl == NULL) { - ifa_free(ifa); - return (EINVAL); + rc = EINVAL; + goto out; } if (len != sdl->sdl_alen) { /* don't allow length to change */ - ifa_free(ifa); - return (EINVAL); + rc = EINVAL; + goto out; } switch (ifp->if_type) { case IFT_ETHER: - case IFT_FDDI: case IFT_XETHER: case IFT_L2VLAN: case IFT_BRIDGE: - case IFT_ARCNET: case IFT_IEEE8023ADLAG: bcopy(lladdr, LLADDR(sdl), len); - ifa_free(ifa); break; default: - ifa_free(ifa); - return (ENODEV); + rc = ENODEV; + goto out; } /* @@ -3683,6 +3864,7 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len) * to re-init it in order to reprogram its * address filter. */ + NET_EPOCH_EXIT(); if ((ifp->if_flags & IFF_UP) != 0) { if (ifp->if_ioctl) { ifp->if_flags &= ~IFF_UP; @@ -3697,6 +3879,9 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len) } EVENTHANDLER_INVOKE(iflladdr_event, ifp); return (0); + out: + NET_EPOCH_EXIT(); + return (rc); } /* @@ -3777,16 +3962,16 @@ if_initname(struct ifnet *ifp, const char *name, int unit) } int -if_printf(struct ifnet *ifp, const char * fmt, ...) +if_printf(struct ifnet *ifp, const char *fmt, ...) { + char if_fmt[256]; va_list ap; - int retval; - retval = printf("%s: ", ifp->if_xname); + snprintf(if_fmt, sizeof(if_fmt), "%s: %s", ifp->if_xname, fmt); va_start(ap, fmt); - retval += vprintf(fmt, ap); + vlog(LOG_INFO, if_fmt, ap); va_end(ap); - return (retval); + return (0); } void @@ -4149,7 +4334,7 @@ if_setupmultiaddr(if_t ifp, void *mta, int *cnt, int max) uint8_t *lmta = (uint8_t *)mta; int mcnt = 0; - TAILQ_FOREACH(ifma, &((struct ifnet *)ifp)->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &((struct ifnet *)ifp)->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; @@ -4184,7 +4369,7 @@ if_multiaddr_count(if_t ifp, int max) count = 0; if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &((struct ifnet *)ifp)->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &((struct ifnet *)ifp)->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; count++; @@ -4202,7 +4387,7 @@ if_multi_apply(struct ifnet *ifp, int (*filter)(void *, struct ifmultiaddr *, in int cnt = 0; if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) cnt += filter(arg, ifma, cnt); if_maddr_runlock(ifp); return (cnt); diff --git a/freebsd/sys/net/if_arc.h b/freebsd/sys/net/if_arc.h deleted file mode 100644 index 8bed38b8..00000000 --- a/freebsd/sys/net/if_arc.h +++ /dev/null @@ -1,145 +0,0 @@ -/* $NetBSD: if_arc.h,v 1.13 1999/11/19 20:41:19 thorpej Exp $ */ -/* $FreeBSD$ */ - -/*- - * SPDX-License-Identifier: BSD-3-Clause - * - * Copyright (c) 1982, 1986, 1993 - * The Regents of the University of California. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * from: NetBSD: if_ether.h,v 1.10 1994/06/29 06:37:55 cgd Exp - * @(#)if_ether.h 8.1 (Berkeley) 6/10/93 - */ - -#ifndef _NET_IF_ARC_H_ -#define _NET_IF_ARC_H_ - -/* - * Arcnet address - 1 octets - * don't know who uses this. - */ -struct arc_addr { - u_int8_t arc_addr_octet[1]; -} __packed; - -/* - * Structure of a 2.5MB/s Arcnet header. - * as given to interface code. - */ -struct arc_header { - u_int8_t arc_shost; - u_int8_t arc_dhost; - u_int8_t arc_type; - /* - * only present for newstyle encoding with LL fragmentation. - * Don't use sizeof(anything), use ARC_HDR{,NEW}LEN instead. - */ - u_int8_t arc_flag; - u_int16_t arc_seqid; - - /* - * only present in exception packets (arc_flag == 0xff) - */ - u_int8_t arc_type2; /* same as arc_type */ - u_int8_t arc_flag2; /* real flag value */ - u_int16_t arc_seqid2; /* real seqid value */ -} __packed; - -#define ARC_ADDR_LEN 1 - -#define ARC_HDRLEN 3 -#define ARC_HDRNEWLEN 6 -#define ARC_HDRNEWLEN_EXC 10 - -/* these lengths are data link layer length - 2 * ARC_ADDR_LEN */ -#define ARC_MIN_LEN 1 -#define ARC_MIN_FORBID_LEN 254 -#define ARC_MAX_FORBID_LEN 256 -#define ARC_MAX_LEN 508 -#define ARC_MAX_DATA 504 - -/* RFC 1051 */ -#define ARCTYPE_IP_OLD 240 /* IP protocol */ -#define ARCTYPE_ARP_OLD 241 /* address resolution protocol */ - -/* RFC 1201 */ -#define ARCTYPE_IP 212 /* IP protocol */ -#define ARCTYPE_ARP 213 /* address resolution protocol */ -#define ARCTYPE_REVARP 214 /* reverse addr resolution protocol */ - -#define ARCTYPE_ATALK 221 /* Appletalk */ -#define ARCTYPE_BANIAN 247 /* Banyan Vines */ -#define ARCTYPE_IPX 250 /* Novell IPX */ - -#define ARCTYPE_INET6 0xc4 /* IPng */ -#define ARCTYPE_DIAGNOSE 0x80 /* as per ANSI/ATA 878.1 */ - -#define ARCMTU 507 -#define ARCMIN 0 - -#define ARC_PHDS_MAXMTU 60480 - -struct arccom { - struct ifnet *ac_ifp; /* network-visible interface */ - - u_int16_t ac_seqid; /* seq. id used by PHDS encap. */ - - u_int8_t arc_shost; - u_int8_t arc_dhost; - u_int8_t arc_type; - - u_int8_t dummy0; - u_int16_t dummy1; - int sflag, fsflag, rsflag; - struct mbuf *curr_frag; - - struct ac_frag { - u_int8_t af_maxflag; /* from first packet */ - u_int8_t af_lastseen; /* last split flag seen */ - u_int16_t af_seqid; - struct mbuf *af_packet; - } ac_fragtab[256]; /* indexed by sender ll address */ -}; - -#ifdef _KERNEL -extern u_int8_t arcbroadcastaddr; -extern int arc_ipmtu; /* XXX new ip only, no RFC 1051! */ - -void arc_ifattach(struct ifnet *, u_int8_t); -void arc_ifdetach(struct ifnet *); -void arc_storelladdr(struct ifnet *, u_int8_t); -int arc_isphds(u_int8_t); -void arc_input(struct ifnet *, struct mbuf *); -int arc_output(struct ifnet *, struct mbuf *, - const struct sockaddr *, struct route *); -int arc_ioctl(struct ifnet *, u_long, caddr_t); - -void arc_frag_init(struct ifnet *); -struct mbuf * arc_frag_next(struct ifnet *); -#endif - -#endif /* _NET_IF_ARC_H_ */ diff --git a/freebsd/sys/net/if_arcsubr.c b/freebsd/sys/net/if_arcsubr.c deleted file mode 100644 index 9f0a03d5..00000000 --- a/freebsd/sys/net/if_arcsubr.c +++ /dev/null @@ -1,833 +0,0 @@ -#include <machine/rtems-bsd-kernel-space.h> - -/* $NetBSD: if_arcsubr.c,v 1.36 2001/06/14 05:44:23 itojun Exp $ */ -/* $FreeBSD$ */ - -/*- - * SPDX-License-Identifier: BSD-4-Clause - * - * Copyright (c) 1994, 1995 Ignatios Souvatzis - * Copyright (c) 1982, 1989, 1993 - * The Regents of the University of California. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the University of - * California, Berkeley and its contributors. - * 4. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * from: NetBSD: if_ethersubr.c,v 1.9 1994/06/29 06:36:11 cgd Exp - * @(#)if_ethersubr.c 8.1 (Berkeley) 6/10/93 - * - */ -#include <rtems/bsd/local/opt_inet.h> -#include <rtems/bsd/local/opt_inet6.h> - -#include <sys/param.h> -#include <sys/systm.h> -#include <sys/kernel.h> -#include <sys/module.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/protosw.h> -#include <sys/socket.h> -#include <sys/sockio.h> -#include <sys/errno.h> -#include <sys/syslog.h> - -#include <machine/cpu.h> - -#include <net/if.h> -#include <net/if_var.h> -#include <net/netisr.h> -#include <net/route.h> -#include <net/if_dl.h> -#include <net/if_types.h> -#include <net/if_arc.h> -#include <net/if_arp.h> -#include <net/bpf.h> -#include <net/if_llatbl.h> - -#if defined(INET) || defined(INET6) -#include <netinet/in.h> -#include <netinet/in_var.h> -#include <netinet/if_ether.h> -#endif - -#ifdef INET6 -#include <netinet6/nd6.h> -#endif - -#define ARCNET_ALLOW_BROKEN_ARP - -static struct mbuf *arc_defrag(struct ifnet *, struct mbuf *); -static int arc_resolvemulti(struct ifnet *, struct sockaddr **, - struct sockaddr *); - -u_int8_t arcbroadcastaddr = 0; - -#define ARC_LLADDR(ifp) (*(u_int8_t *)IF_LLADDR(ifp)) - -#define senderr(e) { error = (e); goto bad;} -#define SIN(s) ((const struct sockaddr_in *)(s)) - -/* - * ARCnet output routine. - * Encapsulate a packet of type family for the local net. - * Assumes that ifp is actually pointer to arccom structure. - */ -int -arc_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, - struct route *ro) -{ - struct arc_header *ah; - int error; - u_int8_t atype, adst; - int loop_copy = 0; - int isphds; -#if defined(INET) || defined(INET6) - int is_gw = 0; -#endif - - if (!((ifp->if_flags & IFF_UP) && - (ifp->if_drv_flags & IFF_DRV_RUNNING))) - return(ENETDOWN); /* m, m1 aren't initialized yet */ - - error = 0; -#if defined(INET) || defined(INET6) - if (ro != NULL) - is_gw = (ro->ro_flags & RT_HAS_GW) != 0; -#endif - - switch (dst->sa_family) { -#ifdef INET - case AF_INET: - - /* - * For now, use the simple IP addr -> ARCnet addr mapping - */ - if (m->m_flags & (M_BCAST|M_MCAST)) - adst = arcbroadcastaddr; /* ARCnet broadcast address */ - else if (ifp->if_flags & IFF_NOARP) - adst = ntohl(SIN(dst)->sin_addr.s_addr) & 0xFF; - else { - error = arpresolve(ifp, is_gw, m, dst, &adst, NULL, - NULL); - if (error) - return (error == EWOULDBLOCK ? 0 : error); - } - - atype = (ifp->if_flags & IFF_LINK0) ? - ARCTYPE_IP_OLD : ARCTYPE_IP; - break; - case AF_ARP: - { - struct arphdr *ah; - ah = mtod(m, struct arphdr *); - ah->ar_hrd = htons(ARPHRD_ARCNET); - - loop_copy = -1; /* if this is for us, don't do it */ - - switch(ntohs(ah->ar_op)) { - case ARPOP_REVREQUEST: - case ARPOP_REVREPLY: - atype = ARCTYPE_REVARP; - break; - case ARPOP_REQUEST: - case ARPOP_REPLY: - default: - atype = ARCTYPE_ARP; - break; - } - - if (m->m_flags & M_BCAST) - bcopy(ifp->if_broadcastaddr, &adst, ARC_ADDR_LEN); - else - bcopy(ar_tha(ah), &adst, ARC_ADDR_LEN); - - } - break; -#endif -#ifdef INET6 - case AF_INET6: - if ((m->m_flags & M_MCAST) != 0) - adst = arcbroadcastaddr; /* ARCnet broadcast address */ - else { - error = nd6_resolve(ifp, is_gw, m, dst, &adst, NULL, - NULL); - if (error != 0) - return (error == EWOULDBLOCK ? 0 : error); - } - atype = ARCTYPE_INET6; - break; -#endif - case AF_UNSPEC: - { - const struct arc_header *ah; - - loop_copy = -1; - ah = (const struct arc_header *)dst->sa_data; - adst = ah->arc_dhost; - atype = ah->arc_type; - - if (atype == ARCTYPE_ARP) { - atype = (ifp->if_flags & IFF_LINK0) ? - ARCTYPE_ARP_OLD: ARCTYPE_ARP; - -#ifdef ARCNET_ALLOW_BROKEN_ARP - /* - * XXX It's not clear per RFC826 if this is needed, but - * "assigned numbers" say this is wrong. - * However, e.g., AmiTCP 3.0Beta used it... we make this - * switchable for emergency cases. Not perfect, but... - */ - if (ifp->if_flags & IFF_LINK2) - mtod(m, struct arphdr *)->ar_pro = atype - 1; -#endif - } - break; - } - default: - if_printf(ifp, "can't handle af%d\n", dst->sa_family); - senderr(EAFNOSUPPORT); - } - - isphds = arc_isphds(atype); - M_PREPEND(m, isphds ? ARC_HDRNEWLEN : ARC_HDRLEN, M_NOWAIT); - if (m == NULL) - senderr(ENOBUFS); - ah = mtod(m, struct arc_header *); - ah->arc_type = atype; - ah->arc_dhost = adst; - ah->arc_shost = ARC_LLADDR(ifp); - if (isphds) { - ah->arc_flag = 0; - ah->arc_seqid = 0; - } - - if ((ifp->if_flags & IFF_SIMPLEX) && (loop_copy != -1)) { - if ((m->m_flags & M_BCAST) || (loop_copy > 0)) { - struct mbuf *n = m_copym(m, 0, M_COPYALL, M_NOWAIT); - - (void) if_simloop(ifp, n, dst->sa_family, ARC_HDRLEN); - } else if (ah->arc_dhost == ah->arc_shost) { - (void) if_simloop(ifp, m, dst->sa_family, ARC_HDRLEN); - return (0); /* XXX */ - } - } - - BPF_MTAP(ifp, m); - - error = ifp->if_transmit(ifp, m); - - return (error); - -bad: - if (m) - m_freem(m); - return (error); -} - -void -arc_frag_init(struct ifnet *ifp) -{ - struct arccom *ac; - - ac = (struct arccom *)ifp->if_l2com; - ac->curr_frag = 0; -} - -struct mbuf * -arc_frag_next(struct ifnet *ifp) -{ - struct arccom *ac; - struct mbuf *m; - struct arc_header *ah; - - ac = (struct arccom *)ifp->if_l2com; - if ((m = ac->curr_frag) == NULL) { - int tfrags; - - /* dequeue new packet */ - IF_DEQUEUE(&ifp->if_snd, m); - if (m == NULL) - return 0; - - ah = mtod(m, struct arc_header *); - if (!arc_isphds(ah->arc_type)) - return m; - - ++ac->ac_seqid; /* make the seqid unique */ - tfrags = howmany(m->m_pkthdr.len, ARC_MAX_DATA); - ac->fsflag = 2 * tfrags - 3; - ac->sflag = 0; - ac->rsflag = ac->fsflag; - ac->arc_dhost = ah->arc_dhost; - ac->arc_shost = ah->arc_shost; - ac->arc_type = ah->arc_type; - - m_adj(m, ARC_HDRNEWLEN); - ac->curr_frag = m; - } - - /* split out next fragment and return it */ - if (ac->sflag < ac->fsflag) { - /* we CAN'T have short packets here */ - ac->curr_frag = m_split(m, ARC_MAX_DATA, M_NOWAIT); - if (ac->curr_frag == 0) { - m_freem(m); - return 0; - } - - M_PREPEND(m, ARC_HDRNEWLEN, M_NOWAIT); - if (m == NULL) { - m_freem(ac->curr_frag); - ac->curr_frag = 0; - return 0; - } - - ah = mtod(m, struct arc_header *); - ah->arc_flag = ac->rsflag; - ah->arc_seqid = ac->ac_seqid; - - ac->sflag += 2; - ac->rsflag = ac->sflag; - } else if ((m->m_pkthdr.len >= - ARC_MIN_FORBID_LEN - ARC_HDRNEWLEN + 2) && - (m->m_pkthdr.len <= - ARC_MAX_FORBID_LEN - ARC_HDRNEWLEN + 2)) { - ac->curr_frag = 0; - - M_PREPEND(m, ARC_HDRNEWLEN_EXC, M_NOWAIT); - if (m == NULL) - return 0; - - ah = mtod(m, struct arc_header *); - ah->arc_flag = 0xFF; - ah->arc_seqid = 0xFFFF; - ah->arc_type2 = ac->arc_type; - ah->arc_flag2 = ac->sflag; - ah->arc_seqid2 = ac->ac_seqid; - } else { - ac->curr_frag = 0; - - M_PREPEND(m, ARC_HDRNEWLEN, M_NOWAIT); - if (m == NULL) - return 0; - - ah = mtod(m, struct arc_header *); - ah->arc_flag = ac->sflag; - ah->arc_seqid = ac->ac_seqid; - } - - ah->arc_dhost = ac->arc_dhost; - ah->arc_shost = ac->arc_shost; - ah->arc_type = ac->arc_type; - - return m; -} - -/* - * Defragmenter. Returns mbuf if last packet found, else - * NULL. frees incoming mbuf as necessary. - */ - -static __inline struct mbuf * -arc_defrag(struct ifnet *ifp, struct mbuf *m) -{ - struct arc_header *ah, *ah1; - struct arccom *ac; - struct ac_frag *af; - struct mbuf *m1; - char *s; - int newflen; - u_char src,dst,typ; - - ac = (struct arccom *)ifp->if_l2com; - - if (m->m_len < ARC_HDRNEWLEN) { - m = m_pullup(m, ARC_HDRNEWLEN); - if (m == NULL) { - if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); - return NULL; - } - } - - ah = mtod(m, struct arc_header *); - typ = ah->arc_type; - - if (!arc_isphds(typ)) - return m; - - src = ah->arc_shost; - dst = ah->arc_dhost; - - if (ah->arc_flag == 0xff) { - m_adj(m, 4); - - if (m->m_len < ARC_HDRNEWLEN) { - m = m_pullup(m, ARC_HDRNEWLEN); - if (m == NULL) { - if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); - return NULL; - } - } - - ah = mtod(m, struct arc_header *); - } - - af = &ac->ac_fragtab[src]; - m1 = af->af_packet; - s = "debug code error"; - - if (ah->arc_flag & 1) { - /* - * first fragment. We always initialize, which is - * about the right thing to do, as we only want to - * accept one fragmented packet per src at a time. - */ - if (m1 != NULL) - m_freem(m1); - - af->af_packet = m; - m1 = m; - af->af_maxflag = ah->arc_flag; - af->af_lastseen = 0; - af->af_seqid = ah->arc_seqid; - - return NULL; - /* notreached */ - } else { - /* check for unfragmented packet */ - if (ah->arc_flag == 0) - return m; - - /* do we have a first packet from that src? */ - if (m1 == NULL) { - s = "no first frag"; - goto outofseq; - } - - ah1 = mtod(m1, struct arc_header *); - - if (ah->arc_seqid != ah1->arc_seqid) { - s = "seqid differs"; - goto outofseq; - } - - if (typ != ah1->arc_type) { - s = "type differs"; - goto outofseq; - } - - if (dst != ah1->arc_dhost) { - s = "dest host differs"; - goto outofseq; - } - - /* typ, seqid and dst are ok here. */ - - if (ah->arc_flag == af->af_lastseen) { - m_freem(m); - return NULL; - } - - if (ah->arc_flag == af->af_lastseen + 2) { - /* ok, this is next fragment */ - af->af_lastseen = ah->arc_flag; - m_adj(m,ARC_HDRNEWLEN); - - /* - * m_cat might free the first mbuf (with pkthdr) - * in 2nd chain; therefore: - */ - - newflen = m->m_pkthdr.len; - - m_cat(m1,m); - - m1->m_pkthdr.len += newflen; - - /* is it the last one? */ - if (af->af_lastseen > af->af_maxflag) { - af->af_packet = NULL; - return(m1); - } else - return NULL; - } - s = "other reason"; - /* if all else fails, it is out of sequence, too */ - } -outofseq: - if (m1) { - m_freem(m1); - af->af_packet = NULL; - } - - if (m) - m_freem(m); - - log(LOG_INFO,"%s: got out of seq. packet: %s\n", - ifp->if_xname, s); - - return NULL; -} - -/* - * return 1 if Packet Header Definition Standard, else 0. - * For now: old IP, old ARP aren't obviously. Lacking correct information, - * we guess that besides new IP and new ARP also IPX and APPLETALK are PHDS. - * (Apple and Novell corporations were involved, among others, in PHDS work). - * Easiest is to assume that everybody else uses that, too. - */ -int -arc_isphds(u_int8_t type) -{ - return (type != ARCTYPE_IP_OLD && - type != ARCTYPE_ARP_OLD && - type != ARCTYPE_DIAGNOSE); -} - -/* - * Process a received Arcnet packet; - * the packet is in the mbuf chain m with - * the ARCnet header. - */ -void -arc_input(struct ifnet *ifp, struct mbuf *m) -{ - struct arc_header *ah; - int isr; - u_int8_t atype; - - if ((ifp->if_flags & IFF_UP) == 0) { - m_freem(m); - return; - } - - /* possibly defragment: */ - m = arc_defrag(ifp, m); - if (m == NULL) - return; - - BPF_MTAP(ifp, m); - - ah = mtod(m, struct arc_header *); - /* does this belong to us? */ - if ((ifp->if_flags & IFF_PROMISC) == 0 - && ah->arc_dhost != arcbroadcastaddr - && ah->arc_dhost != ARC_LLADDR(ifp)) { - m_freem(m); - return; - } - - if_inc_counter(ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len); - - if (ah->arc_dhost == arcbroadcastaddr) { - m->m_flags |= M_BCAST|M_MCAST; - if_inc_counter(ifp, IFCOUNTER_IMCASTS, 1); - } - - atype = ah->arc_type; - switch (atype) { -#ifdef INET - case ARCTYPE_IP: - m_adj(m, ARC_HDRNEWLEN); - isr = NETISR_IP; - break; - - case ARCTYPE_IP_OLD: - m_adj(m, ARC_HDRLEN); - isr = NETISR_IP; - break; - - case ARCTYPE_ARP: - if (ifp->if_flags & IFF_NOARP) { - /* Discard packet if ARP is disabled on interface */ - m_freem(m); - return; - } - m_adj(m, ARC_HDRNEWLEN); - isr = NETISR_ARP; -#ifdef ARCNET_ALLOW_BROKEN_ARP - mtod(m, struct arphdr *)->ar_pro = htons(ETHERTYPE_IP); -#endif - break; - - case ARCTYPE_ARP_OLD: - if (ifp->if_flags & IFF_NOARP) { - /* Discard packet if ARP is disabled on interface */ - m_freem(m); - return; - } - m_adj(m, ARC_HDRLEN); - isr = NETISR_ARP; -#ifdef ARCNET_ALLOW_BROKEN_ARP - mtod(m, struct arphdr *)->ar_pro = htons(ETHERTYPE_IP); -#endif - break; -#endif -#ifdef INET6 - case ARCTYPE_INET6: - m_adj(m, ARC_HDRNEWLEN); - isr = NETISR_IPV6; - break; -#endif - default: - m_freem(m); - return; - } - M_SETFIB(m, ifp->if_fib); - netisr_dispatch(isr, m); -} - -/* - * Register (new) link level address. - */ -void -arc_storelladdr(struct ifnet *ifp, u_int8_t lla) -{ - ARC_LLADDR(ifp) = lla; -} - -/* - * Perform common duties while attaching to interface list - */ -void -arc_ifattach(struct ifnet *ifp, u_int8_t lla) -{ - struct ifaddr *ifa; - struct sockaddr_dl *sdl; - struct arccom *ac; - - if_attach(ifp); - ifp->if_addrlen = 1; - ifp->if_hdrlen = ARC_HDRLEN; - ifp->if_mtu = 1500; - ifp->if_resolvemulti = arc_resolvemulti; - if (ifp->if_baudrate == 0) - ifp->if_baudrate = 2500000; - ifa = ifp->if_addr; - KASSERT(ifa != NULL, ("%s: no lladdr!\n", __func__)); - sdl = (struct sockaddr_dl *)ifa->ifa_addr; - sdl->sdl_type = IFT_ARCNET; - sdl->sdl_alen = ifp->if_addrlen; - - if (ifp->if_flags & IFF_BROADCAST) - ifp->if_flags |= IFF_MULTICAST|IFF_ALLMULTI; - - ac = (struct arccom *)ifp->if_l2com; - ac->ac_seqid = (time_second) & 0xFFFF; /* try to make seqid unique */ - if (lla == 0) { - /* XXX this message isn't entirely clear, to me -- cgd */ - log(LOG_ERR,"%s: link address 0 reserved for broadcasts. Please change it and ifconfig %s down up\n", - ifp->if_xname, ifp->if_xname); - } - arc_storelladdr(ifp, lla); - - ifp->if_broadcastaddr = &arcbroadcastaddr; - - bpfattach(ifp, DLT_ARCNET, ARC_HDRLEN); -} - -void -arc_ifdetach(struct ifnet *ifp) -{ - bpfdetach(ifp); - if_detach(ifp); -} - -int -arc_ioctl(struct ifnet *ifp, u_long command, caddr_t data) -{ - struct ifaddr *ifa = (struct ifaddr *) data; - struct ifreq *ifr = (struct ifreq *) data; - int error = 0; - - switch (command) { - case SIOCSIFADDR: - ifp->if_flags |= IFF_UP; - switch (ifa->ifa_addr->sa_family) { -#ifdef INET - case AF_INET: - ifp->if_init(ifp->if_softc); /* before arpwhohas */ - arp_ifinit(ifp, ifa); - break; -#endif - default: - ifp->if_init(ifp->if_softc); - break; - } - break; - - case SIOCGIFADDR: - ifr->ifr_addr.sa_data[0] = ARC_LLADDR(ifp); - break; - - case SIOCADDMULTI: - case SIOCDELMULTI: - if (ifr == NULL) - error = EAFNOSUPPORT; - else { - switch (ifr->ifr_addr.sa_family) { - case AF_INET: - case AF_INET6: - error = 0; - break; - default: - error = EAFNOSUPPORT; - break; - } - } - break; - - case SIOCSIFMTU: - /* - * Set the interface MTU. - * mtu can't be larger than ARCMTU for RFC1051 - * and can't be larger than ARC_PHDS_MTU - */ - if (((ifp->if_flags & IFF_LINK0) && ifr->ifr_mtu > ARCMTU) || - ifr->ifr_mtu > ARC_PHDS_MAXMTU) - error = EINVAL; - else - ifp->if_mtu = ifr->ifr_mtu; - break; - } - - return (error); -} - -/* based on ether_resolvemulti() */ -int -arc_resolvemulti(struct ifnet *ifp, struct sockaddr **llsa, - struct sockaddr *sa) -{ - struct sockaddr_dl *sdl; -#ifdef INET - struct sockaddr_in *sin; -#endif -#ifdef INET6 - struct sockaddr_in6 *sin6; -#endif - - switch(sa->sa_family) { - case AF_LINK: - /* - * No mapping needed. Just check that it's a valid MC address. - */ - sdl = (struct sockaddr_dl *)sa; - if (*LLADDR(sdl) != arcbroadcastaddr) - return EADDRNOTAVAIL; - *llsa = NULL; - return 0; -#ifdef INET - case AF_INET: - sin = (struct sockaddr_in *)sa; - if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) - return EADDRNOTAVAIL; - sdl = link_init_sdl(ifp, *llsa, IFT_ETHER); - sdl->sdl_alen = ARC_ADDR_LEN; - *LLADDR(sdl) = 0; - *llsa = (struct sockaddr *)sdl; - return 0; -#endif -#ifdef INET6 - case AF_INET6: - sin6 = (struct sockaddr_in6 *)sa; - if (IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) { - /* - * An IP6 address of 0 means listen to all - * of the Ethernet multicast address used for IP6. - * (This is used for multicast routers.) - */ - ifp->if_flags |= IFF_ALLMULTI; - *llsa = NULL; - return 0; - } - if (!IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr)) - return EADDRNOTAVAIL; - sdl = link_init_sdl(ifp, *llsa, IFT_ETHER); - sdl->sdl_alen = ARC_ADDR_LEN; - *LLADDR(sdl) = 0; - *llsa = (struct sockaddr *)sdl; - return 0; -#endif - - default: - /* - * Well, the text isn't quite right, but it's the name - * that counts... - */ - return EAFNOSUPPORT; - } -} - -static MALLOC_DEFINE(M_ARCCOM, "arccom", "ARCNET interface internals"); - -static void* -arc_alloc(u_char type, struct ifnet *ifp) -{ - struct arccom *ac; - - ac = malloc(sizeof(struct arccom), M_ARCCOM, M_WAITOK | M_ZERO); - ac->ac_ifp = ifp; - - return (ac); -} - -static void -arc_free(void *com, u_char type) -{ - - free(com, M_ARCCOM); -} - -static int -arc_modevent(module_t mod, int type, void *data) -{ - - switch (type) { - case MOD_LOAD: - if_register_com_alloc(IFT_ARCNET, arc_alloc, arc_free); - break; - case MOD_UNLOAD: - if_deregister_com_alloc(IFT_ARCNET); - break; - default: - return EOPNOTSUPP; - } - - return (0); -} - -static moduledata_t arc_mod = { - "arcnet", - arc_modevent, - 0 -}; - -DECLARE_MODULE(arcnet, arc_mod, SI_SUB_INIT_IF, SI_ORDER_ANY); -MODULE_VERSION(arcnet, 1); diff --git a/freebsd/sys/net/if_arp.h b/freebsd/sys/net/if_arp.h index 1a66c89c..070dbafe 100644 --- a/freebsd/sys/net/if_arp.h +++ b/freebsd/sys/net/if_arp.h @@ -49,7 +49,6 @@ struct arphdr { u_short ar_hrd; /* format of hardware address */ #define ARPHRD_ETHER 1 /* ethernet hardware format */ #define ARPHRD_IEEE802 6 /* token-ring hardware format */ -#define ARPHRD_ARCNET 7 /* arcnet hardware format */ #define ARPHRD_FRELAY 15 /* frame relay hardware format */ #define ARPHRD_IEEE1394 24 /* firewire hardware format */ #define ARPHRD_INFINIBAND 32 /* infiniband hardware format */ diff --git a/freebsd/sys/net/if_bridge.c b/freebsd/sys/net/if_bridge.c index 248cd45a..3e774934 100644 --- a/freebsd/sys/net/if_bridge.c +++ b/freebsd/sys/net/if_bridge.c @@ -74,8 +74,8 @@ * * - Currently only supports Ethernet-like interfaces (Ethernet, * 802.11, VLANs on Ethernet, etc.) Figure out a nice way - * to bridge other types of interfaces (FDDI-FDDI, and maybe - * consider heterogeneous bridges). + * to bridge other types of interfaces (maybe consider + * heterogeneous bridges). */ #include <sys/cdefs.h> @@ -342,7 +342,6 @@ static int bridge_fragment(struct ifnet *, struct mbuf **mp, static void bridge_linkstate(struct ifnet *ifp); static void bridge_linkcheck(struct bridge_softc *sc); -extern void (*bridge_linkstate_p)(struct ifnet *ifp); /* The default bridge vlan is 1 (IEEE 802.1Q-2003 Table 9-2) */ #define VLANTAGOF(_m) \ @@ -558,10 +557,7 @@ bridge_modevent(module_t mod, int type, void *data) bridge_rtnode_zone = uma_zcreate("bridge_rtnode", sizeof(struct bridge_rtnode), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); - bridge_input_p = bridge_input; - bridge_output_p = bridge_output; bridge_dn_p = bridge_dummynet; - bridge_linkstate_p = bridge_linkstate; bridge_detach_cookie = EVENTHANDLER_REGISTER( ifnet_departure_event, bridge_ifdetach, NULL, EVENTHANDLER_PRI_ANY); @@ -570,10 +566,7 @@ bridge_modevent(module_t mod, int type, void *data) EVENTHANDLER_DEREGISTER(ifnet_departure_event, bridge_detach_cookie); uma_zdestroy(bridge_rtnode_zone); - bridge_input_p = NULL; - bridge_output_p = NULL; bridge_dn_p = NULL; - bridge_linkstate_p = NULL; break; default: return (EOPNOTSUPP); @@ -1043,6 +1036,9 @@ bridge_delete_member(struct bridge_softc *sc, struct bridge_iflist *bif, KASSERT(bif->bif_addrcnt == 0, ("%s: %d bridge routes referenced", __func__, bif->bif_addrcnt)); + ifs->if_bridge_output = NULL; + ifs->if_bridge_input = NULL; + ifs->if_bridge_linkstate = NULL; BRIDGE_UNLOCK(sc); if (!gone) { switch (ifs->if_type) { @@ -1200,6 +1196,9 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg) } ifs->if_bridge = sc; + ifs->if_bridge_output = bridge_output; + ifs->if_bridge_input = bridge_input; + ifs->if_bridge_linkstate = bridge_linkstate; bstp_create(&sc->sc_stp, &bif->bif_stp, bif->bif_ifp); /* * XXX: XLOCK HERE!?! diff --git a/freebsd/sys/net/if_bridgevar.h b/freebsd/sys/net/if_bridgevar.h index 84ff8d21..f4f61706 100644 --- a/freebsd/sys/net/if_bridgevar.h +++ b/freebsd/sys/net/if_bridgevar.h @@ -309,23 +309,20 @@ struct ifbpstpconf { (_sc)->sc_iflist_xcnt--; \ } while (0) -#define BRIDGE_INPUT(_ifp, _m) do { \ - KASSERT(bridge_input_p != NULL, \ +#define BRIDGE_INPUT(_ifp, _m) do { \ + KASSERT((_ifp)->if_bridge_input != NULL, \ ("%s: if_bridge not loaded!", __func__)); \ - _m = (*bridge_input_p)(_ifp, _m); \ + _m = (*(_ifp)->if_bridge_input)(_ifp, _m); \ if (_m != NULL) \ _ifp = _m->m_pkthdr.rcvif; \ } while (0) #define BRIDGE_OUTPUT(_ifp, _m, _err) do { \ - KASSERT(bridge_output_p != NULL, \ + KASSERT((_ifp)->if_bridge_output != NULL, \ ("%s: if_bridge not loaded!", __func__)); \ - _err = (*bridge_output_p)(_ifp, _m, NULL, NULL); \ + _err = (*(_ifp)->if_bridge_output)(_ifp, _m, NULL, NULL); \ } while (0) -extern struct mbuf *(*bridge_input_p)(struct ifnet *, struct mbuf *); -extern int (*bridge_output_p)(struct ifnet *, struct mbuf *, - struct sockaddr *, struct rtentry *); extern void (*bridge_dn_p)(struct mbuf *, struct ifnet *); #endif /* _KERNEL */ diff --git a/freebsd/sys/net/if_clone.c b/freebsd/sys/net/if_clone.c index 295bddf4..5a9c20c2 100644 --- a/freebsd/sys/net/if_clone.c +++ b/freebsd/sys/net/if_clone.c @@ -69,6 +69,7 @@ struct if_clone { char ifc_name[IFCLOSIZ]; /* (c) Name of device, e.g. `gif' */ struct unrhdr *ifc_unrhdr; /* (c) alloc_unr(9) header */ int ifc_maxunit; /* (c) maximum unit number */ + int ifc_flags; long ifc_refcnt; /* (i) Reference count. */ LIST_HEAD(, ifnet) ifc_iflist; /* (i) List of cloned interfaces */ struct mtx ifc_mtx; /* Mutex to protect members. */ @@ -234,7 +235,8 @@ if_clone_createif(struct if_clone *ifc, char *name, size_t len, caddr_t params) if (ifp == NULL) panic("%s: lookup failed for %s", __func__, name); - if_addgroup(ifp, ifc->ifc_name); + if ((ifc->ifc_flags & IFC_NOGROUP) == 0) + if_addgroup(ifp, ifc->ifc_name); IF_CLONE_LOCK(ifc); IFC_IFLIST_INSERT(ifc, ifp); @@ -321,8 +323,8 @@ if_clone_destroyif(struct if_clone *ifc, struct ifnet *ifp) CURVNET_RESTORE(); return (ENXIO); /* ifp is not on the list. */ } - - if_delgroup(ifp, ifc->ifc_name); + if ((ifc->ifc_flags & IFC_NOGROUP) == 0) + if_delgroup(ifp, ifc->ifc_name); if (ifc->ifc_type == SIMPLE) err = ifc_simple_destroy(ifc, ifp); @@ -330,7 +332,8 @@ if_clone_destroyif(struct if_clone *ifc, struct ifnet *ifp) err = (*ifc->ifc_destroy)(ifc, ifp); if (err != 0) { - if_addgroup(ifp, ifc->ifc_name); + if ((ifc->ifc_flags & IFC_NOGROUP) == 0) + if_addgroup(ifp, ifc->ifc_name); IF_CLONE_LOCK(ifc); IFC_IFLIST_INSERT(ifc, ifp); @@ -415,7 +418,7 @@ if_clone_simple(const char *name, ifcs_create_t create, ifcs_destroy_t destroy, for (unit = 0; unit < minifs; unit++) { char name[IFNAMSIZ]; - int error; + int error __unused; snprintf(name, IFNAMSIZ, "%s%d", ifc->ifc_name, unit); error = if_clone_createif(ifc, name, IFNAMSIZ, NULL); @@ -555,9 +558,10 @@ if_clone_findifc(struct ifnet *ifp) void if_clone_addgroup(struct ifnet *ifp, struct if_clone *ifc) { - - if_addgroup(ifp, ifc->ifc_name); - IF_CLONE_REMREF(ifc); + if ((ifc->ifc_flags & IFC_NOGROUP) == 0) { + if_addgroup(ifp, ifc->ifc_name); + IF_CLONE_REMREF(ifc); + } } /* @@ -734,3 +738,21 @@ ifc_simple_destroy(struct if_clone *ifc, struct ifnet *ifp) return (0); } + +const char * +ifc_name(struct if_clone *ifc) +{ + return (ifc->ifc_name); +} + +void +ifc_flags_set(struct if_clone *ifc, int flags) +{ + ifc->ifc_flags = flags; +} + +int +ifc_flags_get(struct if_clone *ifc) +{ + return (ifc->ifc_flags); +} diff --git a/freebsd/sys/net/if_clone.h b/freebsd/sys/net/if_clone.h index f26ab63b..5dceacf6 100644 --- a/freebsd/sys/net/if_clone.h +++ b/freebsd/sys/net/if_clone.h @@ -37,6 +37,8 @@ #ifdef _KERNEL +#define IFC_NOGROUP 0x1 + struct if_clone; /* Methods. */ @@ -59,6 +61,9 @@ void if_clone_detach(struct if_clone *); int ifc_name2unit(const char *name, int *unit); int ifc_alloc_unit(struct if_clone *, int *); void ifc_free_unit(struct if_clone *, int); +const char *ifc_name(struct if_clone *); +void ifc_flags_set(struct if_clone *, int flags); +int ifc_flags_get(struct if_clone *); #ifdef _SYS_EVENTHANDLER_H_ /* Interface clone event. */ diff --git a/freebsd/sys/net/if_epair.c b/freebsd/sys/net/if_epair.c index 106c0a43..d727bc2c 100644 --- a/freebsd/sys/net/if_epair.c +++ b/freebsd/sys/net/if_epair.c @@ -55,10 +55,14 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> +#include <sys/hash.h> +#include <sys/jail.h> #include <sys/kernel.h> +#include <sys/libkern.h> #include <sys/malloc.h> #include <sys/mbuf.h> #include <sys/module.h> +#include <sys/proc.h> #include <sys/refcount.h> #include <sys/queue.h> #include <sys/smp.h> @@ -253,7 +257,7 @@ static void epair_nh_sintr(struct mbuf *m) { struct ifnet *ifp; - struct epair_softc *sc; + struct epair_softc *sc __unused; ifp = m->m_pkthdr.rcvif; (*ifp->if_input)(ifp, m); @@ -298,7 +302,7 @@ epair_nh_drainedcpu(u_int cpuid) IFQ_LOCK(&ifp->if_snd); if (IFQ_IS_EMPTY(&ifp->if_snd)) { - struct epair_softc *sc; + struct epair_softc *sc __unused; STAILQ_REMOVE(&epair_dpcpu->epair_ifp_drain_list, elm, epair_ifp_drain, ifp_next); @@ -339,7 +343,7 @@ epair_remove_ifp_from_draining(struct ifnet *ifp) STAILQ_FOREACH_SAFE(elm, &epair_dpcpu->epair_ifp_drain_list, ifp_next, tvar) { if (ifp == elm->ifp) { - struct epair_softc *sc; + struct epair_softc *sc __unused; STAILQ_REMOVE( &epair_dpcpu->epair_ifp_drain_list, elm, @@ -715,6 +719,9 @@ epair_clone_create(struct if_clone *ifc, char *name, size_t len, caddr_t params) struct ifnet *ifp; char *dp; int error, unit, wildcard; + uint64_t hostid; + uint32_t key[3]; + uint32_t hash; uint8_t eaddr[ETHER_ADDR_LEN]; /* 00:00:00:00:00:00 */ /* @@ -726,14 +733,12 @@ epair_clone_create(struct if_clone *ifc, char *name, size_t len, caddr_t params) if (params) { scb = (struct epair_softc *)params; ifp = scb->ifp; - /* Assign a hopefully unique, locally administered etheraddr. */ - eaddr[0] = 0x02; - eaddr[3] = (ifp->if_index >> 8) & 0xff; - eaddr[4] = ifp->if_index & 0xff; + /* Copy epairNa etheraddr and change the last byte. */ + memcpy(eaddr, scb->oifp->if_hw_addr, ETHER_ADDR_LEN); eaddr[5] = 0x0b; ether_ifattach(ifp, eaddr); /* Correctly set the name for the cloner list. */ - strlcpy(name, scb->ifp->if_xname, len); + strlcpy(name, ifp->if_xname, len); return (0); } @@ -837,10 +842,21 @@ epair_clone_create(struct if_clone *ifc, char *name, size_t len, caddr_t params) ifp->if_init = epair_init; if_setsendqlen(ifp, ifqmaxlen); if_setsendqready(ifp); - /* Assign a hopefully unique, locally administered etheraddr. */ + + /* + * Calculate the etheraddr hashing the hostid and the + * interface index. The result would be hopefully unique + */ + getcredhostid(curthread->td_ucred, (unsigned long *)&hostid); + if (hostid == 0) + arc4rand(&hostid, sizeof(hostid), 0); + key[0] = (uint32_t)ifp->if_index; + key[1] = (uint32_t)(hostid & 0xffffffff); + key[2] = (uint32_t)((hostid >> 32) & 0xfffffffff); + hash = jenkins_hash32(key, 3, 0); + eaddr[0] = 0x02; - eaddr[3] = (ifp->if_index >> 8) & 0xff; - eaddr[4] = ifp->if_index & 0xff; + memcpy(&eaddr[1], &hash, 4); eaddr[5] = 0x0a; ether_ifattach(ifp, eaddr); sca->if_qflush = ifp->if_qflush; diff --git a/freebsd/sys/net/if_ethersubr.c b/freebsd/sys/net/if_ethersubr.c index 78ff2385..3893d331 100644 --- a/freebsd/sys/net/if_ethersubr.c +++ b/freebsd/sys/net/if_ethersubr.c @@ -104,9 +104,6 @@ void (*ng_ether_detach_p)(struct ifnet *ifp); void (*vlan_input_p)(struct ifnet *, struct mbuf *); /* if_bridge(4) support */ -struct mbuf *(*bridge_input_p)(struct ifnet *, struct mbuf *); -int (*bridge_output_p)(struct ifnet *, struct mbuf *, - struct sockaddr *, struct rtentry *); void (*bridge_dn_p)(struct mbuf *, struct ifnet *); /* if_lagg(4) support */ @@ -518,7 +515,7 @@ ether_input_internal(struct ifnet *ifp, struct mbuf *m) } eh = mtod(m, struct ether_header *); etype = ntohs(eh->ether_type); - random_harvest_queue(m, sizeof(*m), 2, RANDOM_NET_ETHER); + random_harvest_queue_ether(m, sizeof(*m), 2); CURVNET_SET_QUIET(ifp->if_vnet); @@ -1130,10 +1127,13 @@ ether_ioctl(struct ifnet *ifp, u_long command, caddr_t data) if (error != 0) break; if (ifr->ifr_lan_pcp > 7 && - ifr->ifr_lan_pcp != IFNET_PCP_NONE) + ifr->ifr_lan_pcp != IFNET_PCP_NONE) { error = EINVAL; - else + } else { ifp->if_pcp = ifr->ifr_lan_pcp; + /* broadcast event about PCP change */ + EVENTHANDLER_INVOKE(ifnet_event, ifp, IFNET_EVENT_PCP); + } break; case SIOCGLANPCP: diff --git a/freebsd/sys/net/if_fddisubr.c b/freebsd/sys/net/if_fddisubr.c deleted file mode 100644 index c011346e..00000000 --- a/freebsd/sys/net/if_fddisubr.c +++ /dev/null @@ -1,669 +0,0 @@ -#include <machine/rtems-bsd-kernel-space.h> - -/*- - * SPDX-License-Identifier: BSD-4-Clause - * - * Copyright (c) 1995, 1996 - * Matt Thomas <matt@3am-software.com>. All rights reserved. - * Copyright (c) 1982, 1989, 1993 - * The Regents of the University of California. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the University of - * California, Berkeley and its contributors. - * 4. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * from: if_ethersubr.c,v 1.5 1994/12/13 22:31:45 wollman Exp - * $FreeBSD$ - */ - -#include <rtems/bsd/local/opt_inet.h> -#include <rtems/bsd/local/opt_inet6.h> - -#include <sys/param.h> -#include <sys/systm.h> -#include <sys/kernel.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/module.h> -#include <sys/socket.h> -#include <sys/sockio.h> - -#include <net/if.h> -#include <net/if_var.h> -#include <net/if_dl.h> -#include <net/if_llc.h> -#include <net/if_types.h> -#include <net/if_llatbl.h> - -#include <net/ethernet.h> -#include <net/netisr.h> -#include <net/route.h> -#include <net/bpf.h> -#include <net/fddi.h> - -#if defined(INET) || defined(INET6) -#include <netinet/in.h> -#include <netinet/in_var.h> -#include <netinet/if_ether.h> -#endif -#ifdef INET6 -#include <netinet6/nd6.h> -#endif - -#ifdef DECNET -#include <netdnet/dn.h> -#endif - -#include <security/mac/mac_framework.h> - -static const u_char fddibroadcastaddr[FDDI_ADDR_LEN] = - { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; - -static int fddi_resolvemulti(struct ifnet *, struct sockaddr **, - struct sockaddr *); -static int fddi_output(struct ifnet *, struct mbuf *, const struct sockaddr *, - struct route *); -static void fddi_input(struct ifnet *ifp, struct mbuf *m); - -#define senderr(e) do { error = (e); goto bad; } while (0) - -/* - * FDDI output routine. - * Encapsulate a packet of type family for the local net. - * Use trailer local net encapsulation if enough data in first - * packet leaves a multiple of 512 bytes of data in remainder. - */ -static int -fddi_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, - struct route *ro) -{ - u_int16_t type; - int loop_copy = 0, error = 0, hdrcmplt = 0; - u_char esrc[FDDI_ADDR_LEN], edst[FDDI_ADDR_LEN]; - struct fddi_header *fh; -#if defined(INET) || defined(INET6) - int is_gw = 0; -#endif - -#ifdef MAC - error = mac_ifnet_check_transmit(ifp, m); - if (error) - senderr(error); -#endif - - if (ifp->if_flags & IFF_MONITOR) - senderr(ENETDOWN); - if (!((ifp->if_flags & IFF_UP) && - (ifp->if_drv_flags & IFF_DRV_RUNNING))) - senderr(ENETDOWN); - getmicrotime(&ifp->if_lastchange); - -#if defined(INET) || defined(INET6) - if (ro != NULL) - is_gw = (ro->ro_flags & RT_HAS_GW) != 0; -#endif - - switch (dst->sa_family) { -#ifdef INET - case AF_INET: { - error = arpresolve(ifp, is_gw, m, dst, edst, NULL, NULL); - if (error) - return (error == EWOULDBLOCK ? 0 : error); - type = htons(ETHERTYPE_IP); - break; - } - case AF_ARP: - { - struct arphdr *ah; - ah = mtod(m, struct arphdr *); - ah->ar_hrd = htons(ARPHRD_ETHER); - - loop_copy = -1; /* if this is for us, don't do it */ - - switch (ntohs(ah->ar_op)) { - case ARPOP_REVREQUEST: - case ARPOP_REVREPLY: - type = htons(ETHERTYPE_REVARP); - break; - case ARPOP_REQUEST: - case ARPOP_REPLY: - default: - type = htons(ETHERTYPE_ARP); - break; - } - - if (m->m_flags & M_BCAST) - bcopy(ifp->if_broadcastaddr, edst, FDDI_ADDR_LEN); - else - bcopy(ar_tha(ah), edst, FDDI_ADDR_LEN); - - } - break; -#endif /* INET */ -#ifdef INET6 - case AF_INET6: - error = nd6_resolve(ifp, is_gw, m, dst, edst, NULL, NULL); - if (error) - return (error == EWOULDBLOCK ? 0 : error); - type = htons(ETHERTYPE_IPV6); - break; -#endif /* INET6 */ - case pseudo_AF_HDRCMPLT: - { - const struct ether_header *eh; - - hdrcmplt = 1; - eh = (const struct ether_header *)dst->sa_data; - bcopy(eh->ether_shost, esrc, FDDI_ADDR_LEN); - /* FALLTHROUGH */ - } - - case AF_UNSPEC: - { - const struct ether_header *eh; - - loop_copy = -1; - eh = (const struct ether_header *)dst->sa_data; - bcopy(eh->ether_dhost, edst, FDDI_ADDR_LEN); - if (*edst & 1) - m->m_flags |= (M_BCAST|M_MCAST); - type = eh->ether_type; - break; - } - - case AF_IMPLINK: - { - fh = mtod(m, struct fddi_header *); - error = EPROTONOSUPPORT; - switch (fh->fddi_fc & (FDDIFC_C|FDDIFC_L|FDDIFC_F)) { - case FDDIFC_LLC_ASYNC: { - /* legal priorities are 0 through 7 */ - if ((fh->fddi_fc & FDDIFC_Z) > 7) - goto bad; - break; - } - case FDDIFC_LLC_SYNC: { - /* FDDIFC_Z bits reserved, must be zero */ - if (fh->fddi_fc & FDDIFC_Z) - goto bad; - break; - } - case FDDIFC_SMT: { - /* FDDIFC_Z bits must be non zero */ - if ((fh->fddi_fc & FDDIFC_Z) == 0) - goto bad; - break; - } - default: { - /* anything else is too dangerous */ - goto bad; - } - } - error = 0; - if (fh->fddi_dhost[0] & 1) - m->m_flags |= (M_BCAST|M_MCAST); - goto queue_it; - } - default: - if_printf(ifp, "can't handle af%d\n", dst->sa_family); - senderr(EAFNOSUPPORT); - } - - /* - * Add LLC header. - */ - if (type != 0) { - struct llc *l; - M_PREPEND(m, LLC_SNAPFRAMELEN, M_NOWAIT); - if (m == NULL) - senderr(ENOBUFS); - l = mtod(m, struct llc *); - l->llc_control = LLC_UI; - l->llc_dsap = l->llc_ssap = LLC_SNAP_LSAP; - l->llc_snap.org_code[0] = - l->llc_snap.org_code[1] = - l->llc_snap.org_code[2] = 0; - l->llc_snap.ether_type = htons(type); - } - - /* - * Add local net header. If no space in first mbuf, - * allocate another. - */ - M_PREPEND(m, FDDI_HDR_LEN, M_NOWAIT); - if (m == NULL) - senderr(ENOBUFS); - fh = mtod(m, struct fddi_header *); - fh->fddi_fc = FDDIFC_LLC_ASYNC|FDDIFC_LLC_PRIO4; - bcopy((caddr_t)edst, (caddr_t)fh->fddi_dhost, FDDI_ADDR_LEN); - queue_it: - if (hdrcmplt) - bcopy((caddr_t)esrc, (caddr_t)fh->fddi_shost, FDDI_ADDR_LEN); - else - bcopy(IF_LLADDR(ifp), (caddr_t)fh->fddi_shost, - FDDI_ADDR_LEN); - - /* - * If a simplex interface, and the packet is being sent to our - * Ethernet address or a broadcast address, loopback a copy. - * XXX To make a simplex device behave exactly like a duplex - * device, we should copy in the case of sending to our own - * ethernet address (thus letting the original actually appear - * on the wire). However, we don't do that here for security - * reasons and compatibility with the original behavior. - */ - if ((ifp->if_flags & IFF_SIMPLEX) && (loop_copy != -1)) { - if ((m->m_flags & M_BCAST) || (loop_copy > 0)) { - struct mbuf *n; - n = m_copym(m, 0, M_COPYALL, M_NOWAIT); - (void) if_simloop(ifp, n, dst->sa_family, - FDDI_HDR_LEN); - } else if (bcmp(fh->fddi_dhost, fh->fddi_shost, - FDDI_ADDR_LEN) == 0) { - (void) if_simloop(ifp, m, dst->sa_family, - FDDI_HDR_LEN); - return (0); /* XXX */ - } - } - - error = (ifp->if_transmit)(ifp, m); - if (error) - if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - - return (error); - -bad: - if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - if (m) - m_freem(m); - return (error); -} - -/* - * Process a received FDDI packet. - */ -static void -fddi_input(ifp, m) - struct ifnet *ifp; - struct mbuf *m; -{ - int isr; - struct llc *l; - struct fddi_header *fh; - - /* - * Do consistency checks to verify assumptions - * made by code past this point. - */ - if ((m->m_flags & M_PKTHDR) == 0) { - if_printf(ifp, "discard frame w/o packet header\n"); - if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); - m_freem(m); - return; - } - if (m->m_pkthdr.rcvif == NULL) { - if_printf(ifp, "discard frame w/o interface pointer\n"); - if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); - m_freem(m); - return; - } - - m = m_pullup(m, FDDI_HDR_LEN); - if (m == NULL) { - if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); - goto dropanyway; - } - fh = mtod(m, struct fddi_header *); - - /* - * Discard packet if interface is not up. - */ - if (!((ifp->if_flags & IFF_UP) && - (ifp->if_drv_flags & IFF_DRV_RUNNING))) - goto dropanyway; - - /* - * Give bpf a chance at the packet. - */ - BPF_MTAP(ifp, m); - - /* - * Interface marked for monitoring; discard packet. - */ - if (ifp->if_flags & IFF_MONITOR) { - m_freem(m); - return; - } - -#ifdef MAC - mac_ifnet_create_mbuf(ifp, m); -#endif - - /* - * Update interface statistics. - */ - if_inc_counter(ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len); - getmicrotime(&ifp->if_lastchange); - - /* - * Discard non local unicast packets when interface - * is in promiscuous mode. - */ - if ((ifp->if_flags & IFF_PROMISC) && ((fh->fddi_dhost[0] & 1) == 0) && - (bcmp(IF_LLADDR(ifp), (caddr_t)fh->fddi_dhost, - FDDI_ADDR_LEN) != 0)) - goto dropanyway; - - /* - * Set mbuf flags for bcast/mcast. - */ - if (fh->fddi_dhost[0] & 1) { - if (bcmp(ifp->if_broadcastaddr, fh->fddi_dhost, - FDDI_ADDR_LEN) == 0) - m->m_flags |= M_BCAST; - else - m->m_flags |= M_MCAST; - if_inc_counter(ifp, IFCOUNTER_IMCASTS, 1); - } - -#ifdef M_LINK0 - /* - * If this has a LLC priority of 0, then mark it so upper - * layers have a hint that it really came via a FDDI/Ethernet - * bridge. - */ - if ((fh->fddi_fc & FDDIFC_LLC_PRIO7) == FDDIFC_LLC_PRIO0) - m->m_flags |= M_LINK0; -#endif - - /* Strip off FDDI header. */ - m_adj(m, FDDI_HDR_LEN); - - m = m_pullup(m, LLC_SNAPFRAMELEN); - if (m == NULL) { - if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); - goto dropanyway; - } - l = mtod(m, struct llc *); - - switch (l->llc_dsap) { - case LLC_SNAP_LSAP: - { - u_int16_t type; - if ((l->llc_control != LLC_UI) || - (l->llc_ssap != LLC_SNAP_LSAP)) { - if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1); - goto dropanyway; - } - if (l->llc_snap.org_code[0] != 0 || - l->llc_snap.org_code[1] != 0 || - l->llc_snap.org_code[2] != 0) { - if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1); - goto dropanyway; - } - - type = ntohs(l->llc_snap.ether_type); - m_adj(m, LLC_SNAPFRAMELEN); - - switch (type) { -#ifdef INET - case ETHERTYPE_IP: - isr = NETISR_IP; - break; - - case ETHERTYPE_ARP: - if (ifp->if_flags & IFF_NOARP) - goto dropanyway; - isr = NETISR_ARP; - break; -#endif -#ifdef INET6 - case ETHERTYPE_IPV6: - isr = NETISR_IPV6; - break; -#endif -#ifdef DECNET - case ETHERTYPE_DECNET: - isr = NETISR_DECNET; - break; -#endif - default: - /* printf("fddi_input: unknown protocol 0x%x\n", type); */ - if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1); - goto dropanyway; - } - break; - } - - default: - /* printf("fddi_input: unknown dsap 0x%x\n", l->llc_dsap); */ - if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1); - goto dropanyway; - } - M_SETFIB(m, ifp->if_fib); - netisr_dispatch(isr, m); - return; - -dropanyway: - if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); - if (m) - m_freem(m); - return; -} - -/* - * Perform common duties while attaching to interface list - */ -void -fddi_ifattach(ifp, lla, bpf) - struct ifnet *ifp; - const u_int8_t *lla; - int bpf; -{ - struct ifaddr *ifa; - struct sockaddr_dl *sdl; - - ifp->if_type = IFT_FDDI; - ifp->if_addrlen = FDDI_ADDR_LEN; - ifp->if_hdrlen = 21; - - if_attach(ifp); /* Must be called before additional assignments */ - - ifp->if_mtu = FDDIMTU; - ifp->if_output = fddi_output; - ifp->if_input = fddi_input; - ifp->if_resolvemulti = fddi_resolvemulti; - ifp->if_broadcastaddr = fddibroadcastaddr; - ifp->if_baudrate = 100000000; -#ifdef IFF_NOTRAILERS - ifp->if_flags |= IFF_NOTRAILERS; -#endif - ifa = ifp->if_addr; - KASSERT(ifa != NULL, ("%s: no lladdr!\n", __func__)); - - sdl = (struct sockaddr_dl *)ifa->ifa_addr; - sdl->sdl_type = IFT_FDDI; - sdl->sdl_alen = ifp->if_addrlen; - bcopy(lla, LLADDR(sdl), ifp->if_addrlen); - - if (bpf) - bpfattach(ifp, DLT_FDDI, FDDI_HDR_LEN); - - return; -} - -void -fddi_ifdetach(ifp, bpf) - struct ifnet *ifp; - int bpf; -{ - - if (bpf) - bpfdetach(ifp); - - if_detach(ifp); - - return; -} - -int -fddi_ioctl (ifp, command, data) - struct ifnet *ifp; - u_long command; - caddr_t data; -{ - struct ifaddr *ifa; - struct ifreq *ifr; - int error; - - ifa = (struct ifaddr *) data; - ifr = (struct ifreq *) data; - error = 0; - - switch (command) { - case SIOCSIFADDR: - ifp->if_flags |= IFF_UP; - - switch (ifa->ifa_addr->sa_family) { -#ifdef INET - case AF_INET: /* before arpwhohas */ - ifp->if_init(ifp->if_softc); - arp_ifinit(ifp, ifa); - break; -#endif - default: - ifp->if_init(ifp->if_softc); - break; - } - break; - case SIOCGIFADDR: - bcopy(IF_LLADDR(ifp), &ifr->ifr_addr.sa_data[0], - FDDI_ADDR_LEN); - break; - case SIOCSIFMTU: - /* - * Set the interface MTU. - */ - if (ifr->ifr_mtu > FDDIMTU) { - error = EINVAL; - } else { - ifp->if_mtu = ifr->ifr_mtu; - } - break; - default: - error = EINVAL; - break; - } - - return (error); -} - -static int -fddi_resolvemulti(ifp, llsa, sa) - struct ifnet *ifp; - struct sockaddr **llsa; - struct sockaddr *sa; -{ - struct sockaddr_dl *sdl; -#ifdef INET - struct sockaddr_in *sin; -#endif -#ifdef INET6 - struct sockaddr_in6 *sin6; -#endif - u_char *e_addr; - - switch(sa->sa_family) { - case AF_LINK: - /* - * No mapping needed. Just check that it's a valid MC address. - */ - sdl = (struct sockaddr_dl *)sa; - e_addr = LLADDR(sdl); - if ((e_addr[0] & 1) != 1) - return (EADDRNOTAVAIL); - *llsa = NULL; - return (0); - -#ifdef INET - case AF_INET: - sin = (struct sockaddr_in *)sa; - if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) - return (EADDRNOTAVAIL); - sdl = link_init_sdl(ifp, *llsa, IFT_FDDI); - sdl->sdl_nlen = 0; - sdl->sdl_alen = FDDI_ADDR_LEN; - sdl->sdl_slen = 0; - e_addr = LLADDR(sdl); - ETHER_MAP_IP_MULTICAST(&sin->sin_addr, e_addr); - *llsa = (struct sockaddr *)sdl; - return (0); -#endif -#ifdef INET6 - case AF_INET6: - sin6 = (struct sockaddr_in6 *)sa; - if (IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) { - /* - * An IP6 address of 0 means listen to all - * of the Ethernet multicast address used for IP6. - * (This is used for multicast routers.) - */ - ifp->if_flags |= IFF_ALLMULTI; - *llsa = NULL; - return (0); - } - if (!IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr)) - return (EADDRNOTAVAIL); - sdl = link_init_sdl(ifp, *llsa, IFT_FDDI); - sdl->sdl_nlen = 0; - sdl->sdl_alen = FDDI_ADDR_LEN; - sdl->sdl_slen = 0; - e_addr = LLADDR(sdl); - ETHER_MAP_IPV6_MULTICAST(&sin6->sin6_addr, e_addr); - *llsa = (struct sockaddr *)sdl; - return (0); -#endif - - default: - /* - * Well, the text isn't quite right, but it's the name - * that counts... - */ - return (EAFNOSUPPORT); - } - - return (0); -} - -static moduledata_t fddi_mod = { - "fddi", /* module name */ - NULL, /* event handler */ - 0 /* extra data */ -}; - -DECLARE_MODULE(fddi, fddi_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); -MODULE_VERSION(fddi, 1); diff --git a/freebsd/sys/net/if_ipsec.c b/freebsd/sys/net/if_ipsec.c index 91afe2ab..eaeecd5a 100644 --- a/freebsd/sys/net/if_ipsec.c +++ b/freebsd/sys/net/if_ipsec.c @@ -440,7 +440,7 @@ ipsec_if_input(struct mbuf *m, struct secasvar *sav, uint32_t af) m->m_pkthdr.rcvif = ifp; IPSEC_SC_RUNLOCK(); - /* m_clrprotoflags(m); */ + m_clrprotoflags(m); M_SETFIB(m, ifp->if_fib); BPF_MTAP2(ifp, &af, sizeof(af), m); if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); diff --git a/freebsd/sys/net/if_lagg.c b/freebsd/sys/net/if_lagg.c index 36c8095a..578078c2 100644 --- a/freebsd/sys/net/if_lagg.c +++ b/freebsd/sys/net/if_lagg.c @@ -75,6 +75,18 @@ __FBSDID("$FreeBSD$"); #include <net/if_lagg.h> #include <net/ieee8023ad_lacp.h> +#define LAGG_RLOCK() epoch_enter_preempt(net_epoch_preempt) +#define LAGG_RUNLOCK() epoch_exit_preempt(net_epoch_preempt) +#define LAGG_RLOCK_ASSERT() MPASS(in_epoch()) +#define LAGG_UNLOCK_ASSERT() MPASS(!in_epoch()) + +#define LAGG_SX_INIT(_sc) sx_init(&(_sc)->sc_sx, "if_lagg sx") +#define LAGG_SX_DESTROY(_sc) sx_destroy(&(_sc)->sc_sx) +#define LAGG_XLOCK(_sc) sx_xlock(&(_sc)->sc_sx) +#define LAGG_XUNLOCK(_sc) sx_xunlock(&(_sc)->sc_sx) +#define LAGG_SXLOCK_ASSERT(_sc) sx_assert(&(_sc)->sc_sx, SA_LOCKED) +#define LAGG_XLOCK_ASSERT(_sc) sx_assert(&(_sc)->sc_sx, SA_XLOCKED) + /* Special flags we should propagate to the lagg ports. */ static struct { int flag; @@ -336,14 +348,11 @@ lagg_proto_detach(struct lagg_softc *sc) lagg_proto pr; LAGG_XLOCK_ASSERT(sc); - LAGG_WLOCK_ASSERT(sc); pr = sc->sc_proto; sc->sc_proto = LAGG_PROTO_NONE; if (lagg_protos[pr].pr_detach != NULL) lagg_protos[pr].pr_detach(sc); - else - LAGG_WUNLOCK(sc); } static int @@ -439,10 +448,10 @@ lagg_register_vlan(void *arg, struct ifnet *ifp, u_int16_t vtag) if (ifp->if_softc != arg) /* Not our event */ return; - LAGG_SLOCK(sc); - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) + LAGG_RLOCK(); + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) EVENTHANDLER_INVOKE(vlan_config, lp->lp_ifp, vtag); - LAGG_SUNLOCK(sc); + LAGG_RUNLOCK(); } /* @@ -458,10 +467,10 @@ lagg_unregister_vlan(void *arg, struct ifnet *ifp, u_int16_t vtag) if (ifp->if_softc != arg) /* Not our event */ return; - LAGG_SLOCK(sc); - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) + LAGG_RLOCK(); + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) EVENTHANDLER_INVOKE(vlan_unconfig, lp->lp_ifp, vtag); - LAGG_SUNLOCK(sc); + LAGG_RUNLOCK(); } static int @@ -477,7 +486,6 @@ lagg_clone_create(struct if_clone *ifc, int unit, caddr_t params) free(sc, M_DEVBUF); return (ENOSPC); } - LAGG_LOCK_INIT(sc); LAGG_SX_INIT(sc); LAGG_XLOCK(sc); @@ -490,7 +498,7 @@ lagg_clone_create(struct if_clone *ifc, int unit, caddr_t params) lagg_proto_attach(sc, LAGG_PROTO_DEFAULT); - SLIST_INIT(&sc->sc_ports); + CK_SLIST_INIT(&sc->sc_ports); /* Initialise pseudo media types */ ifmedia_init(&sc->sc_media, 0, lagg_media_change, @@ -548,13 +556,11 @@ lagg_clone_destroy(struct ifnet *ifp) EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); /* Shutdown and remove lagg ports */ - while ((lp = SLIST_FIRST(&sc->sc_ports)) != NULL) + while ((lp = CK_SLIST_FIRST(&sc->sc_ports)) != NULL) lagg_port_destroy(lp, 1); /* Unhook the aggregation protocol */ - LAGG_WLOCK(sc); lagg_proto_detach(sc); - LAGG_UNLOCK_ASSERT(sc); LAGG_XUNLOCK(sc); ifmedia_removeall(&sc->sc_media); @@ -566,7 +572,6 @@ lagg_clone_destroy(struct ifnet *ifp) LAGG_LIST_UNLOCK(); LAGG_SX_DESTROY(sc); - LAGG_LOCK_DESTROY(sc); free(sc, M_DEVBUF); } @@ -582,7 +587,7 @@ lagg_capabilities(struct lagg_softc *sc) /* Get common enabled capabilities for the lagg ports */ ena = ~0; - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) ena &= lp->lp_ifp->if_capenable; ena = (ena == ~0 ? 0 : ena); @@ -592,7 +597,7 @@ lagg_capabilities(struct lagg_softc *sc) */ do { pena = ena; - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { lagg_setcaps(lp, ena); ena &= lp->lp_ifp->if_capenable; } @@ -602,7 +607,7 @@ lagg_capabilities(struct lagg_softc *sc) cap = ~0; hwa = ~(uint64_t)0; memset(&hw_tsomax, 0, sizeof(hw_tsomax)); - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { cap &= lp->lp_ifp->if_capabilities; hwa &= lp->lp_ifp->if_hwassist; if_hw_tsomax_common(lp->lp_ifp, &hw_tsomax); @@ -653,7 +658,7 @@ lagg_port_create(struct lagg_softc *sc, struct ifnet *ifp) return (EPROTONOSUPPORT); /* Allow the first Ethernet member to define the MTU */ - if (SLIST_EMPTY(&sc->sc_ports)) + if (CK_SLIST_EMPTY(&sc->sc_ports)) sc->sc_ifp->if_mtu = ifp->if_mtu; else if (sc->sc_ifp->if_mtu != ifp->if_mtu) { if_printf(sc->sc_ifp, "invalid MTU for %s\n", @@ -690,19 +695,16 @@ lagg_port_create(struct lagg_softc *sc, struct ifnet *ifp) bcopy(IF_LLADDR(ifp), lp->lp_lladdr, ETHER_ADDR_LEN); lp->lp_ifcapenable = ifp->if_capenable; - if (SLIST_EMPTY(&sc->sc_ports)) { - LAGG_WLOCK(sc); + if (CK_SLIST_EMPTY(&sc->sc_ports)) { bcopy(IF_LLADDR(ifp), IF_LLADDR(sc->sc_ifp), ETHER_ADDR_LEN); lagg_proto_lladdr(sc); - LAGG_WUNLOCK(sc); EVENTHANDLER_INVOKE(iflladdr_event, sc->sc_ifp); } else { if_setlladdr(ifp, IF_LLADDR(sc->sc_ifp), ETHER_ADDR_LEN); } lagg_setflags(lp, 1); - LAGG_WLOCK(sc); - if (SLIST_EMPTY(&sc->sc_ports)) + if (CK_SLIST_EMPTY(&sc->sc_ports)) sc->sc_primary = lp; /* Change the interface type */ @@ -725,28 +727,27 @@ lagg_port_create(struct lagg_softc *sc, struct ifnet *ifp) * is predictable and `ifconfig laggN create ...` command * will lead to the same result each time. */ - SLIST_FOREACH(tlp, &sc->sc_ports, lp_entries) { + LAGG_RLOCK(); + CK_SLIST_FOREACH(tlp, &sc->sc_ports, lp_entries) { if (tlp->lp_ifp->if_index < ifp->if_index && ( - SLIST_NEXT(tlp, lp_entries) == NULL || - SLIST_NEXT(tlp, lp_entries)->lp_ifp->if_index > + CK_SLIST_NEXT(tlp, lp_entries) == NULL || + ((struct lagg_port*)CK_SLIST_NEXT(tlp, lp_entries))->lp_ifp->if_index > ifp->if_index)) break; } + LAGG_RUNLOCK(); if (tlp != NULL) - SLIST_INSERT_AFTER(tlp, lp, lp_entries); + CK_SLIST_INSERT_AFTER(tlp, lp, lp_entries); else - SLIST_INSERT_HEAD(&sc->sc_ports, lp, lp_entries); + CK_SLIST_INSERT_HEAD(&sc->sc_ports, lp, lp_entries); sc->sc_count++; lagg_setmulti(lp); - LAGG_WUNLOCK(sc); if ((error = lagg_proto_addport(sc, lp)) != 0) { /* Remove the port, without calling pr_delport. */ - LAGG_WLOCK(sc); lagg_port_destroy(lp, 0); - LAGG_UNLOCK_ASSERT(sc); return (error); } @@ -766,7 +767,7 @@ lagg_port_checkstacking(struct lagg_softc *sc) int m = 0; LAGG_SXLOCK_ASSERT(sc); - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { if (lp->lp_flags & LAGG_PORT_STACK) { sc_ptr = (struct lagg_softc *)lp->lp_ifp->if_softc; m = MAX(m, lagg_port_checkstacking(sc_ptr)); @@ -777,6 +778,19 @@ lagg_port_checkstacking(struct lagg_softc *sc) } #endif +static void +lagg_port_destroy_cb(epoch_context_t ec) +{ + struct lagg_port *lp; + struct ifnet *ifp; + + lp = __containerof(ec, struct lagg_port, lp_epoch_ctx); + ifp = lp->lp_ifp; + + if_rele(ifp); + free(lp, M_DEVBUF); +} + static int lagg_port_destroy(struct lagg_port *lp, int rundelport) { @@ -788,11 +802,8 @@ lagg_port_destroy(struct lagg_port *lp, int rundelport) LAGG_XLOCK_ASSERT(sc); - if (rundelport) { - LAGG_WLOCK(sc); + if (rundelport) lagg_proto_delport(sc, lp); - } else - LAGG_WLOCK_ASSERT(sc); if (lp->lp_detaching == 0) lagg_clrmulti(lp); @@ -811,14 +822,14 @@ lagg_port_destroy(struct lagg_port *lp, int rundelport) } /* Finally, remove the port from the lagg */ - SLIST_REMOVE(&sc->sc_ports, lp, lagg_port, lp_entries); + CK_SLIST_REMOVE(&sc->sc_ports, lp, lagg_port, lp_entries); sc->sc_count--; /* Update the primary interface */ if (lp == sc->sc_primary) { uint8_t lladdr[ETHER_ADDR_LEN]; - if ((lp0 = SLIST_FIRST(&sc->sc_ports)) == NULL) + if ((lp0 = CK_SLIST_FIRST(&sc->sc_ports)) == NULL) bzero(&lladdr, ETHER_ADDR_LEN); else bcopy(lp0->lp_lladdr, lladdr, ETHER_ADDR_LEN); @@ -826,19 +837,16 @@ lagg_port_destroy(struct lagg_port *lp, int rundelport) if (sc->sc_destroying == 0) { bcopy(lladdr, IF_LLADDR(sc->sc_ifp), ETHER_ADDR_LEN); lagg_proto_lladdr(sc); - LAGG_WUNLOCK(sc); EVENTHANDLER_INVOKE(iflladdr_event, sc->sc_ifp); - } else - LAGG_WUNLOCK(sc); + } /* * Update lladdr for each port (new primary needs update * as well, to switch from old lladdr to its 'real' one) */ - SLIST_FOREACH(lp_ptr, &sc->sc_ports, lp_entries) + CK_SLIST_FOREACH(lp_ptr, &sc->sc_ports, lp_entries) if_setlladdr(lp_ptr->lp_ifp, lladdr, ETHER_ADDR_LEN); - } else - LAGG_WUNLOCK(sc); + } if (lp->lp_ifflags) if_printf(ifp, "%s: lp_ifflags unclean\n", __func__); @@ -849,9 +857,11 @@ lagg_port_destroy(struct lagg_port *lp, int rundelport) if_setlladdr(ifp, lp->lp_lladdr, ETHER_ADDR_LEN); } - if_rele(ifp); - free(lp, M_DEVBUF); - + /* + * free port and release it's ifnet reference after a grace period has + * elapsed. + */ + epoch_call(net_epoch_preempt, &lp->lp_epoch_ctx, lagg_port_destroy_cb); /* Update lagg capabilities */ lagg_capabilities(sc); lagg_linkstate(sc); @@ -880,15 +890,15 @@ lagg_port_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) break; } - LAGG_SLOCK(sc); + LAGG_RLOCK(); if ((lp = ifp->if_lagg) == NULL || lp->lp_softc != sc) { error = ENOENT; - LAGG_SUNLOCK(sc); + LAGG_RUNLOCK(); break; } lagg_port2req(lp, rp); - LAGG_SUNLOCK(sc); + LAGG_RUNLOCK(); break; case SIOCSIFCAP: @@ -944,17 +954,16 @@ lagg_get_counter(struct ifnet *ifp, ift_counter cnt) struct lagg_softc *sc; struct lagg_port *lp; struct ifnet *lpifp; - struct rm_priotracker tracker; uint64_t newval, oldval, vsum; /* Revise this when we've got non-generic counters. */ KASSERT(cnt < IFCOUNTERS, ("%s: invalid cnt %d", __func__, cnt)); sc = (struct lagg_softc *)ifp->if_softc; - LAGG_RLOCK(sc, &tracker); vsum = 0; - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { + LAGG_RLOCK(); + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { /* Saved attached value */ oldval = lp->port_counters.val[cnt]; /* current value */ @@ -963,6 +972,7 @@ lagg_get_counter(struct ifnet *ifp, ift_counter cnt) /* Calculate diff and save new */ vsum += newval - oldval; } + LAGG_RUNLOCK(); /* * Add counter data which might be added by upper @@ -975,7 +985,6 @@ lagg_get_counter(struct ifnet *ifp, ift_counter cnt) */ vsum += sc->detached_counters.val[cnt]; - LAGG_RUNLOCK(sc, &tracker); return (vsum); } @@ -1081,7 +1090,7 @@ lagg_init(void *xsc) * This might be if_setlladdr() notification * that lladdr has been changed. */ - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { if (memcmp(IF_LLADDR(ifp), IF_LLADDR(lp->lp_ifp), ETHER_ADDR_LEN) != 0) if_setlladdr(lp->lp_ifp, IF_LLADDR(ifp), ETHER_ADDR_LEN); @@ -1126,7 +1135,7 @@ lagg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) switch (cmd) { case SIOCGLAGG: - LAGG_SLOCK(sc); + LAGG_XLOCK(sc); buflen = sc->sc_count * sizeof(struct lagg_reqport); outbuf = malloc(buflen, M_TEMP, M_WAITOK | M_ZERO); ra->ra_proto = sc->sc_proto; @@ -1134,7 +1143,7 @@ lagg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) count = 0; buf = outbuf; len = min(ra->ra_size, buflen); - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { if (len < sizeof(rpbuf)) break; @@ -1144,7 +1153,7 @@ lagg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) buf += sizeof(rpbuf); len -= sizeof(rpbuf); } - LAGG_SUNLOCK(sc); + LAGG_XUNLOCK(sc); ra->ra_ports = count; ra->ra_size = count * sizeof(rpbuf); error = copyout(outbuf, ra->ra_port, ra->ra_size); @@ -1160,14 +1169,13 @@ lagg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) } LAGG_XLOCK(sc); - LAGG_WLOCK(sc); lagg_proto_detach(sc); - LAGG_UNLOCK_ASSERT(sc); + LAGG_UNLOCK_ASSERT(); lagg_proto_attach(sc, ra->ra_proto); LAGG_XUNLOCK(sc); break; case SIOCGLAGGOPTS: - LAGG_SLOCK(sc); + LAGG_XLOCK(sc); ro->ro_opts = sc->sc_opts; if (sc->sc_proto == LAGG_PROTO_LACP) { struct lacp_softc *lsc; @@ -1185,13 +1193,13 @@ lagg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) ro->ro_active = sc->sc_active; } else { ro->ro_active = 0; - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) ro->ro_active += LAGG_PORTACTIVE(lp); } ro->ro_bkt = sc->sc_bkt; ro->ro_flapping = sc->sc_flapping; ro->ro_flowid_shift = sc->flowid_shift; - LAGG_SUNLOCK(sc); + LAGG_XUNLOCK(sc); break; case SIOCSLAGGOPTS: if (sc->sc_proto == LAGG_PROTO_ROUNDROBIN) { @@ -1298,14 +1306,14 @@ lagg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) break; case SIOCGLAGGFLAGS: rf->rf_flags = 0; - LAGG_SLOCK(sc); + LAGG_XLOCK(sc); if (sc->sc_flags & MBUF_HASHFLAG_L2) rf->rf_flags |= LAGG_F_HASHL2; if (sc->sc_flags & MBUF_HASHFLAG_L3) rf->rf_flags |= LAGG_F_HASHL3; if (sc->sc_flags & MBUF_HASHFLAG_L4) rf->rf_flags |= LAGG_F_HASHL4; - LAGG_SUNLOCK(sc); + LAGG_XUNLOCK(sc); break; case SIOCSLAGGHASH: error = priv_check(td, PRIV_NET_LAGG); @@ -1332,17 +1340,17 @@ lagg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) break; } - LAGG_SLOCK(sc); + LAGG_RLOCK(); if ((lp = (struct lagg_port *)tpif->if_lagg) == NULL || lp->lp_softc != sc) { error = ENOENT; - LAGG_SUNLOCK(sc); + LAGG_RUNLOCK(); if_rele(tpif); break; } lagg_port2req(lp, rp); - LAGG_SUNLOCK(sc); + LAGG_RUNLOCK(); if_rele(tpif); break; case SIOCSLAGGPORT: @@ -1407,7 +1415,7 @@ lagg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) case SIOCSIFFLAGS: /* Set flags on ports too */ LAGG_XLOCK(sc); - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { lagg_setflags(lp, 1); } @@ -1432,12 +1440,12 @@ lagg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) break; case SIOCADDMULTI: case SIOCDELMULTI: - LAGG_WLOCK(sc); - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { + LAGG_XLOCK(sc); + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { lagg_clrmulti(lp); lagg_setmulti(lp); } - LAGG_WUNLOCK(sc); + LAGG_XUNLOCK(sc); error = 0; break; case SIOCSIFMEDIA: @@ -1447,7 +1455,7 @@ lagg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) case SIOCSIFCAP: LAGG_XLOCK(sc); - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { if (lp->lp_ioctl != NULL) (*lp->lp_ioctl)(lp->lp_ifp, cmd, data); } @@ -1525,9 +1533,8 @@ lagg_setmulti(struct lagg_port *lp) struct ifmultiaddr *ifma; int error; - LAGG_WLOCK_ASSERT(sc); IF_ADDR_WLOCK(scifp); - TAILQ_FOREACH(ifma, &scifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &scifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; mc = malloc(sizeof(struct lagg_mc), M_DEVBUF, M_NOWAIT); @@ -1556,7 +1563,7 @@ lagg_clrmulti(struct lagg_port *lp) { struct lagg_mc *mc; - LAGG_WLOCK_ASSERT(lp->lp_softc); + LAGG_XLOCK_ASSERT(lp->lp_softc); while ((mc = SLIST_FIRST(&lp->lp_mc_head)) != NULL) { SLIST_REMOVE(&lp->lp_mc_head, mc, lagg_mc, mc_entries); if (mc->mc_ifma && lp->lp_detaching == 0) @@ -1636,16 +1643,12 @@ static int lagg_transmit(struct ifnet *ifp, struct mbuf *m) { struct lagg_softc *sc = (struct lagg_softc *)ifp->if_softc; - int error, len, mcast; - struct rm_priotracker tracker; - - len = m->m_pkthdr.len; - mcast = (m->m_flags & (M_MCAST | M_BCAST)) ? 1 : 0; + int error; - LAGG_RLOCK(sc, &tracker); + LAGG_RLOCK(); /* We need a Tx algorithm and at least one port */ if (sc->sc_proto == LAGG_PROTO_NONE || sc->sc_count == 0) { - LAGG_RUNLOCK(sc, &tracker); + LAGG_RUNLOCK(); m_freem(m); if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); return (ENXIO); @@ -1654,7 +1657,7 @@ lagg_transmit(struct ifnet *ifp, struct mbuf *m) ETHER_BPF_MTAP(ifp, m); error = lagg_proto_start(sc, m); - LAGG_RUNLOCK(sc, &tracker); + LAGG_RUNLOCK(); if (error != 0) if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); @@ -1676,33 +1679,25 @@ lagg_input(struct ifnet *ifp, struct mbuf *m) struct lagg_port *lp = ifp->if_lagg; struct lagg_softc *sc = lp->lp_softc; struct ifnet *scifp = sc->sc_ifp; - struct rm_priotracker tracker; - LAGG_RLOCK(sc, &tracker); + LAGG_RLOCK(); if ((scifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || - (lp->lp_flags & LAGG_PORT_DISABLED) || + lp->lp_detaching != 0 || sc->sc_proto == LAGG_PROTO_NONE) { - LAGG_RUNLOCK(sc, &tracker); + LAGG_RUNLOCK(); m_freem(m); return (NULL); } ETHER_BPF_MTAP(scifp, m); - if (lp->lp_detaching != 0) { + m = lagg_proto_input(sc, lp, m); + if (m != NULL && (scifp->if_flags & IFF_MONITOR) != 0) { m_freem(m); m = NULL; - } else - m = lagg_proto_input(sc, lp, m); - - if (m != NULL) { - if (scifp->if_flags & IFF_MONITOR) { - m_freem(m); - m = NULL; - } } - LAGG_RUNLOCK(sc, &tracker); + LAGG_RUNLOCK(); return (m); } @@ -1727,12 +1722,12 @@ lagg_media_status(struct ifnet *ifp, struct ifmediareq *imr) imr->ifm_status = IFM_AVALID; imr->ifm_active = IFM_ETHER | IFM_AUTO; - LAGG_SLOCK(sc); - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { + LAGG_RLOCK(); + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { if (LAGG_PORTACTIVE(lp)) imr->ifm_status |= IFM_ACTIVE; } - LAGG_SUNLOCK(sc); + LAGG_RUNLOCK(); } static void @@ -1745,12 +1740,14 @@ lagg_linkstate(struct lagg_softc *sc) LAGG_XLOCK_ASSERT(sc); /* Our link is considered up if at least one of our ports is active */ - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { + LAGG_RLOCK(); + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { if (lp->lp_ifp->if_link_state == LINK_STATE_UP) { new_link = LINK_STATE_UP; break; } } + LAGG_RUNLOCK(); if_link_state_change(sc->sc_ifp, new_link); /* Update if_baudrate to reflect the max possible speed */ @@ -1763,8 +1760,10 @@ lagg_linkstate(struct lagg_softc *sc) case LAGG_PROTO_LOADBALANCE: case LAGG_PROTO_BROADCAST: speed = 0; - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) + LAGG_RLOCK(); + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) speed += lp->lp_ifp->if_baudrate; + LAGG_RUNLOCK(); sc->sc_ifp->if_baudrate = speed; break; case LAGG_PROTO_LACP: @@ -1805,20 +1804,22 @@ lagg_link_active(struct lagg_softc *sc, struct lagg_port *lp) rval = lp; goto found; } - if ((lp_next = SLIST_NEXT(lp, lp_entries)) != NULL && + if ((lp_next = CK_SLIST_NEXT(lp, lp_entries)) != NULL && LAGG_PORTACTIVE(lp_next)) { rval = lp_next; goto found; } -search: - SLIST_FOREACH(lp_next, &sc->sc_ports, lp_entries) { + search: + LAGG_RLOCK(); + CK_SLIST_FOREACH(lp_next, &sc->sc_ports, lp_entries) { if (LAGG_PORTACTIVE(lp_next)) { + LAGG_RUNLOCK(); rval = lp_next; goto found; } } - + LAGG_RUNLOCK(); found: return (rval); } @@ -1859,10 +1860,10 @@ lagg_rr_start(struct lagg_softc *sc, struct mbuf *m) p = atomic_fetchadd_32(&sc->sc_seq, 1); p %= sc->sc_count; - lp = SLIST_FIRST(&sc->sc_ports); + lp = CK_SLIST_FIRST(&sc->sc_ports); while (p--) - lp = SLIST_NEXT(lp, lp_entries); + lp = CK_SLIST_NEXT(lp, lp_entries); /* * Check the port's link state. This will return the next active @@ -1900,7 +1901,8 @@ lagg_bcast_start(struct lagg_softc *sc, struct mbuf *m) struct lagg_port *lp, *last = NULL; struct mbuf *m0; - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { + LAGG_RLOCK(); + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { if (!LAGG_PORTACTIVE(lp)) continue; @@ -1920,6 +1922,8 @@ lagg_bcast_start(struct lagg_softc *sc, struct mbuf *m) } last = lp; } + LAGG_RUNLOCK(); + if (last == NULL) { m_freem(m); return (ENOENT); @@ -2003,11 +2007,12 @@ lagg_lb_attach(struct lagg_softc *sc) struct lagg_port *lp; struct lagg_lb *lb; + LAGG_XLOCK_ASSERT(sc); lb = malloc(sizeof(struct lagg_lb), M_DEVBUF, M_WAITOK | M_ZERO); lb->lb_key = m_ether_tcpip_hash_init(); sc->sc_psc = lb; - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) lagg_lb_port_create(lp); } @@ -2017,7 +2022,6 @@ lagg_lb_detach(struct lagg_softc *sc) struct lagg_lb *lb; lb = (struct lagg_lb *)sc->sc_psc; - LAGG_WUNLOCK(sc); if (lb != NULL) free(lb, M_DEVBUF); } @@ -2030,7 +2034,8 @@ lagg_lb_porttable(struct lagg_softc *sc, struct lagg_port *lp) int i = 0; bzero(&lb->lb_ports, sizeof(lb->lb_ports)); - SLIST_FOREACH(lp_next, &sc->sc_ports, lp_entries) { + LAGG_RLOCK(); + CK_SLIST_FOREACH(lp_next, &sc->sc_ports, lp_entries) { if (lp_next == lp) continue; if (i >= LAGG_MAX_PORTS) @@ -2040,6 +2045,7 @@ lagg_lb_porttable(struct lagg_softc *sc, struct lagg_port *lp) sc->sc_ifname, lp_next->lp_ifp->if_xname, i); lb->lb_ports[i++] = lp_next; } + LAGG_RUNLOCK(); return (0); } @@ -2106,7 +2112,8 @@ lagg_lacp_attach(struct lagg_softc *sc) struct lagg_port *lp; lacp_attach(sc); - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) + LAGG_XLOCK_ASSERT(sc); + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) lacp_port_create(lp); } @@ -2116,13 +2123,12 @@ lagg_lacp_detach(struct lagg_softc *sc) struct lagg_port *lp; void *psc; - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) + LAGG_XLOCK_ASSERT(sc); + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) lacp_port_destroy(lp); psc = sc->sc_psc; sc->sc_psc = NULL; - LAGG_WUNLOCK(sc); - lacp_detach(psc); } @@ -2134,11 +2140,11 @@ lagg_lacp_lladdr(struct lagg_softc *sc) LAGG_SXLOCK_ASSERT(sc); /* purge all the lacp ports */ - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) lacp_port_destroy(lp); /* add them back in */ - SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) + CK_SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) lacp_port_create(lp); } diff --git a/freebsd/sys/net/if_lagg.h b/freebsd/sys/net/if_lagg.h index 201d909a..f1e2d8f4 100644 --- a/freebsd/sys/net/if_lagg.h +++ b/freebsd/sys/net/if_lagg.h @@ -42,9 +42,8 @@ #define LAGG_PORT_ACTIVE 0x00000004 /* port is active */ #define LAGG_PORT_COLLECTING 0x00000008 /* port is receiving frames */ #define LAGG_PORT_DISTRIBUTING 0x00000010 /* port is sending frames */ -#define LAGG_PORT_DISABLED 0x00000020 /* port is disabled */ #define LAGG_PORT_BITS "\20\01MASTER\02STACK\03ACTIVE\04COLLECTING" \ - "\05DISTRIBUTING\06DISABLED" + "\05DISTRIBUTING" /* Supported lagg PROTOs */ typedef enum { @@ -218,7 +217,7 @@ struct lagg_softc { uint32_t sc_flags; int sc_destroying; /* destroying lagg */ - SLIST_HEAD(__tplhd, lagg_port) sc_ports; /* list of interfaces */ + CK_SLIST_HEAD(__tplhd, lagg_port) sc_ports; /* list of interfaces */ SLIST_ENTRY(lagg_softc) sc_entries; eventhandler_tag vlan_attach; @@ -252,29 +251,10 @@ struct lagg_port { const struct sockaddr *, struct route *); struct lagg_counters port_counters; /* ifp counters copy */ - SLIST_ENTRY(lagg_port) lp_entries; + CK_SLIST_ENTRY(lagg_port) lp_entries; + struct epoch_context lp_epoch_ctx; }; -#define LAGG_LOCK_INIT(_sc) rm_init(&(_sc)->sc_mtx, "if_lagg rmlock") -#define LAGG_LOCK_DESTROY(_sc) rm_destroy(&(_sc)->sc_mtx) -#define LAGG_RLOCK(_sc, _p) rm_rlock(&(_sc)->sc_mtx, (_p)) -#define LAGG_WLOCK(_sc) rm_wlock(&(_sc)->sc_mtx) -#define LAGG_RUNLOCK(_sc, _p) rm_runlock(&(_sc)->sc_mtx, (_p)) -#define LAGG_WUNLOCK(_sc) rm_wunlock(&(_sc)->sc_mtx) -#define LAGG_RLOCK_ASSERT(_sc) rm_assert(&(_sc)->sc_mtx, RA_RLOCKED) -#define LAGG_WLOCK_ASSERT(_sc) rm_assert(&(_sc)->sc_mtx, RA_WLOCKED) -#define LAGG_UNLOCK_ASSERT(_sc) rm_assert(&(_sc)->sc_mtx, RA_UNLOCKED) - -#define LAGG_SX_INIT(_sc) sx_init(&(_sc)->sc_sx, "if_lagg sx") -#define LAGG_SX_DESTROY(_sc) sx_destroy(&(_sc)->sc_sx) -#define LAGG_SLOCK(_sc) sx_slock(&(_sc)->sc_sx) -#define LAGG_XLOCK(_sc) sx_xlock(&(_sc)->sc_sx) -#define LAGG_SUNLOCK(_sc) sx_sunlock(&(_sc)->sc_sx) -#define LAGG_XUNLOCK(_sc) sx_xunlock(&(_sc)->sc_sx) -#define LAGG_SXLOCK_ASSERT(_sc) sx_assert(&(_sc)->sc_sx, SA_LOCKED) -#define LAGG_SLOCK_ASSERT(_sc) sx_assert(&(_sc)->sc_sx, SA_SLOCKED) -#define LAGG_XLOCK_ASSERT(_sc) sx_assert(&(_sc)->sc_sx, SA_XLOCKED) - extern struct mbuf *(*lagg_input_p)(struct ifnet *, struct mbuf *); extern void (*lagg_linkstate_p)(struct ifnet *, int ); diff --git a/freebsd/sys/net/if_llatbl.c b/freebsd/sys/net/if_llatbl.c index d98153b9..d6e9dbaf 100644 --- a/freebsd/sys/net/if_llatbl.c +++ b/freebsd/sys/net/if_llatbl.c @@ -148,7 +148,7 @@ htable_foreach_lle(struct lltable *llt, llt_foreach_cb_t *f, void *farg) error = 0; for (i = 0; i < llt->llt_hsize; i++) { - LIST_FOREACH_SAFE(lle, &llt->lle_head[i], lle_next, next) { + CK_LIST_FOREACH_SAFE(lle, &llt->lle_head[i], lle_next, next) { error = f(llt, lle, farg); if (error != 0) break; @@ -175,7 +175,7 @@ htable_link_entry(struct lltable *llt, struct llentry *lle) lle->lle_tbl = llt; lle->lle_head = lleh; lle->la_flags |= LLE_LINKED; - LIST_INSERT_HEAD(lleh, lle, lle_next); + CK_LIST_INSERT_HEAD(lleh, lle, lle_next); } static void @@ -184,7 +184,7 @@ htable_unlink_entry(struct llentry *lle) if ((lle->la_flags & LLE_LINKED) != 0) { IF_AFDATA_WLOCK_ASSERT(lle->lle_tbl->llt_ifp); - LIST_REMOVE(lle, lle_next); + CK_LIST_REMOVE(lle, lle_next); lle->la_flags &= ~(LLE_VALID | LLE_LINKED); #if 0 lle->lle_tbl = NULL; @@ -209,7 +209,7 @@ htable_prefix_free_cb(struct lltable *llt, struct llentry *lle, void *farg) if (llt->llt_match_prefix(pmd->addr, pmd->mask, pmd->flags, lle)) { LLE_WLOCK(lle); - LIST_INSERT_HEAD(&pmd->dchain, lle, lle_chain); + CK_LIST_INSERT_HEAD(&pmd->dchain, lle, lle_chain); } return (0); @@ -226,7 +226,7 @@ htable_prefix_free(struct lltable *llt, const struct sockaddr *addr, pmd.addr = addr; pmd.mask = mask; pmd.flags = flags; - LIST_INIT(&pmd.dchain); + CK_LIST_INIT(&pmd.dchain); IF_AFDATA_WLOCK(llt->llt_ifp); /* Push matching lles to chain */ @@ -235,7 +235,7 @@ htable_prefix_free(struct lltable *llt, const struct sockaddr *addr, llentries_unlink(llt, &pmd.dchain); IF_AFDATA_WUNLOCK(llt->llt_ifp); - LIST_FOREACH_SAFE(lle, &pmd.dchain, lle_chain, next) + CK_LIST_FOREACH_SAFE(lle, &pmd.dchain, lle_chain, next) lltable_free_entry(llt, lle); } @@ -252,7 +252,7 @@ llentries_unlink(struct lltable *llt, struct llentries *head) { struct llentry *lle, *next; - LIST_FOREACH_SAFE(lle, head, lle_chain, next) + CK_LIST_FOREACH_SAFE(lle, head, lle_chain, next) llt->llt_unlink_entry(lle); } @@ -498,7 +498,7 @@ lltable_free_cb(struct lltable *llt, struct llentry *lle, void *farg) dchain = (struct llentries *)farg; LLE_WLOCK(lle); - LIST_INSERT_HEAD(dchain, lle, lle_chain); + CK_LIST_INSERT_HEAD(dchain, lle, lle_chain); return (0); } @@ -516,14 +516,14 @@ lltable_free(struct lltable *llt) lltable_unlink(llt); - LIST_INIT(&dchain); + CK_LIST_INIT(&dchain); IF_AFDATA_WLOCK(llt->llt_ifp); /* Push all lles to @dchain */ lltable_foreach_lle(llt, lltable_free_cb, &dchain); llentries_unlink(llt, &dchain); IF_AFDATA_WUNLOCK(llt->llt_ifp); - LIST_FOREACH_SAFE(lle, &dchain, lle_chain, next) { + CK_LIST_FOREACH_SAFE(lle, &dchain, lle_chain, next) { if (callout_stop(&lle->lle_timer) > 0) LLE_REMREF(lle); llentry_free(lle); @@ -546,7 +546,7 @@ lltable_drain(int af) continue; for (i=0; i < llt->llt_hsize; i++) { - LIST_FOREACH(lle, &llt->lle_head[i], lle_next) { + CK_LIST_FOREACH(lle, &llt->lle_head[i], lle_next) { LLE_WLOCK(lle); if (lle->la_hold) { m_freem(lle->la_hold); @@ -622,7 +622,7 @@ lltable_allocate_htbl(uint32_t hsize) M_LLTABLE, M_WAITOK | M_ZERO); for (i = 0; i < llt->llt_hsize; i++) - LIST_INIT(&llt->lle_head[i]); + CK_LIST_INIT(&llt->lle_head[i]); /* Set some default callbacks */ llt->llt_link_entry = htable_link_entry; @@ -848,7 +848,7 @@ llatbl_lle_show(struct llentry_sa *la) lle = &la->base; db_printf("lle=%p\n", lle); - db_printf(" lle_next=%p\n", lle->lle_next.le_next); + db_printf(" lle_next=%p\n", lle->lle_next.cle_next); db_printf(" lle_lock=%p\n", &lle->lle_lock); db_printf(" lle_tbl=%p\n", lle->lle_tbl); db_printf(" lle_head=%p\n", lle->lle_head); @@ -919,7 +919,7 @@ llatbl_llt_show(struct lltable *llt) llt, llt->llt_af, llt->llt_ifp); for (i = 0; i < llt->llt_hsize; i++) { - LIST_FOREACH(lle, &llt->lle_head[i], lle_next) { + CK_LIST_FOREACH(lle, &llt->lle_head[i], lle_next) { llatbl_lle_show((struct llentry_sa *)lle); if (db_pager_quit) diff --git a/freebsd/sys/net/if_llatbl.h b/freebsd/sys/net/if_llatbl.h index 9cec4ac7..74301284 100644 --- a/freebsd/sys/net/if_llatbl.h +++ b/freebsd/sys/net/if_llatbl.h @@ -34,13 +34,15 @@ __FBSDID("$FreeBSD$"); #include <sys/_rwlock.h> #include <netinet/in.h> +#include <sys/epoch.h> +#include <sys/ck.h> struct ifnet; struct sysctl_req; struct rt_msghdr; struct rt_addrinfo; struct llentry; -LIST_HEAD(llentries, llentry); +CK_LIST_HEAD(llentries, llentry); #define LLE_MAX_LINKHDR 24 /* Full IB header */ /* @@ -48,7 +50,7 @@ LIST_HEAD(llentries, llentry); * a shared lock */ struct llentry { - LIST_ENTRY(llentry) lle_next; + CK_LIST_ENTRY(llentry) lle_next; union { struct in_addr addr4; struct in6_addr addr6; @@ -76,10 +78,11 @@ struct llentry { int lle_refcnt; char *ll_addr; /* link-layer address */ - LIST_ENTRY(llentry) lle_chain; /* chain of deleted items */ + CK_LIST_ENTRY(llentry) lle_chain; /* chain of deleted items */ struct callout lle_timer; struct rwlock lle_lock; struct mtx req_mtx; + struct epoch_context lle_epoch_ctx; }; #define LLE_WLOCK(lle) rw_wlock(&(lle)->lle_lock) diff --git a/freebsd/sys/net/if_loop.c b/freebsd/sys/net/if_loop.c index bae46891..988b9f9d 100644 --- a/freebsd/sys/net/if_loop.c +++ b/freebsd/sys/net/if_loop.c @@ -138,7 +138,7 @@ lo_clone_create(struct if_clone *ifc, int unit, caddr_t params) ifp->if_output = looutput; ifp->if_snd.ifq_maxlen = ifqmaxlen; ifp->if_capabilities = ifp->if_capenable = - IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6; + IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 | IFCAP_LINKSTATE; ifp->if_hwassist = LO_CSUM_FEATURES | LO_CSUM_FEATURES6; if_attach(ifp); bpfattach(ifp, DLT_NULL, sizeof(u_int32_t)); @@ -415,6 +415,8 @@ loioctl(struct ifnet *ifp, u_long cmd, caddr_t data) break; case SIOCSIFFLAGS: + if_link_state_change(ifp, (ifp->if_flags & IFF_UP) ? + LINK_STATE_UP: LINK_STATE_DOWN); break; case SIOCSIFCAP: diff --git a/freebsd/sys/net/if_media.c b/freebsd/sys/net/if_media.c index cceb0079..867f7fc3 100644 --- a/freebsd/sys/net/if_media.c +++ b/freebsd/sys/net/if_media.c @@ -401,18 +401,6 @@ struct ifmedia_description ifm_subtype_ethernet_descriptions[] = struct ifmedia_description ifm_subtype_ethernet_option_descriptions[] = IFM_SUBTYPE_ETHERNET_OPTION_DESCRIPTIONS; -struct ifmedia_description ifm_subtype_tokenring_descriptions[] = - IFM_SUBTYPE_TOKENRING_DESCRIPTIONS; - -struct ifmedia_description ifm_subtype_tokenring_option_descriptions[] = - IFM_SUBTYPE_TOKENRING_OPTION_DESCRIPTIONS; - -struct ifmedia_description ifm_subtype_fddi_descriptions[] = - IFM_SUBTYPE_FDDI_DESCRIPTIONS; - -struct ifmedia_description ifm_subtype_fddi_option_descriptions[] = - IFM_SUBTYPE_FDDI_OPTION_DESCRIPTIONS; - struct ifmedia_description ifm_subtype_ieee80211_descriptions[] = IFM_SUBTYPE_IEEE80211_DESCRIPTIONS; @@ -448,16 +436,6 @@ struct ifmedia_type_to_subtype ifmedia_types_to_subtypes[] = { NULL, }, { - &ifm_subtype_tokenring_descriptions[0], - &ifm_subtype_tokenring_option_descriptions[0], - NULL, - }, - { - &ifm_subtype_fddi_descriptions[0], - &ifm_subtype_fddi_option_descriptions[0], - NULL, - }, - { &ifm_subtype_ieee80211_descriptions[0], &ifm_subtype_ieee80211_option_descriptions[0], &ifm_subtype_ieee80211_mode_descriptions[0] diff --git a/freebsd/sys/net/if_media.h b/freebsd/sys/net/if_media.h index ecc2b418..97cd6552 100644 --- a/freebsd/sys/net/if_media.h +++ b/freebsd/sys/net/if_media.h @@ -217,32 +217,6 @@ uint64_t ifmedia_baudrate(int); #define IFM_ETH_XSHIFT 6 /* shift XTYPE next to TMASK */ /* - * Token ring - */ -#define IFM_TOKEN 0x00000040 -#define IFM_TOK_STP4 3 /* Shielded twisted pair 4m - DB9 */ -#define IFM_TOK_STP16 4 /* Shielded twisted pair 16m - DB9 */ -#define IFM_TOK_UTP4 5 /* Unshielded twisted pair 4m - RJ45 */ -#define IFM_TOK_UTP16 6 /* Unshielded twisted pair 16m - RJ45 */ -#define IFM_TOK_STP100 7 /* Shielded twisted pair 100m - DB9 */ -#define IFM_TOK_UTP100 8 /* Unshielded twisted pair 100m - RJ45 */ -#define IFM_TOK_ETR 0x00000200 /* Early token release */ -#define IFM_TOK_SRCRT 0x00000400 /* Enable source routing features */ -#define IFM_TOK_ALLR 0x00000800 /* All routes / Single route bcast */ -#define IFM_TOK_DTR 0x00002000 /* Dedicated token ring */ -#define IFM_TOK_CLASSIC 0x00004000 /* Classic token ring */ -#define IFM_TOK_AUTO 0x00008000 /* Automatic Dedicate/Classic token ring */ - -/* - * FDDI - */ -#define IFM_FDDI 0x00000060 -#define IFM_FDDI_SMF 3 /* Single-mode fiber */ -#define IFM_FDDI_MMF 4 /* Multi-mode fiber */ -#define IFM_FDDI_UTP 5 /* CDDI / UTP */ -#define IFM_FDDI_DA 0x00000100 /* Dual attach / single attach */ - -/* * IEEE 802.11 Wireless */ #define IFM_IEEE80211 0x00000080 @@ -393,8 +367,6 @@ struct ifmedia_description { #define IFM_TYPE_DESCRIPTIONS { \ { IFM_ETHER, "Ethernet" }, \ - { IFM_TOKEN, "Token ring" }, \ - { IFM_FDDI, "FDDI" }, \ { IFM_IEEE80211, "IEEE 802.11 Wireless Ethernet" }, \ { IFM_ATM, "ATM" }, \ { 0, NULL }, \ @@ -511,55 +483,6 @@ struct ifmedia_description { { 0, NULL }, \ } -#define IFM_SUBTYPE_TOKENRING_DESCRIPTIONS { \ - { IFM_TOK_STP4, "DB9/4Mbit" }, \ - { IFM_TOK_STP16, "DB9/16Mbit" }, \ - { IFM_TOK_UTP4, "UTP/4Mbit" }, \ - { IFM_TOK_UTP16, "UTP/16Mbit" }, \ - { IFM_TOK_STP100, "STP/100Mbit" }, \ - { IFM_TOK_UTP100, "UTP/100Mbit" }, \ - { 0, NULL }, \ -} - -#define IFM_SUBTYPE_TOKENRING_ALIASES { \ - { IFM_TOK_STP4, "4STP" }, \ - { IFM_TOK_STP16, "16STP" }, \ - { IFM_TOK_UTP4, "4UTP" }, \ - { IFM_TOK_UTP16, "16UTP" }, \ - { IFM_TOK_STP100, "100STP" }, \ - { IFM_TOK_UTP100, "100UTP" }, \ - { 0, NULL }, \ -} - -#define IFM_SUBTYPE_TOKENRING_OPTION_DESCRIPTIONS { \ - { IFM_TOK_ETR, "EarlyTokenRelease" }, \ - { IFM_TOK_SRCRT, "SourceRouting" }, \ - { IFM_TOK_ALLR, "AllRoutes" }, \ - { IFM_TOK_DTR, "Dedicated" }, \ - { IFM_TOK_CLASSIC,"Classic" }, \ - { IFM_TOK_AUTO, " " }, \ - { 0, NULL }, \ -} - -#define IFM_SUBTYPE_FDDI_DESCRIPTIONS { \ - { IFM_FDDI_SMF, "Single-mode" }, \ - { IFM_FDDI_MMF, "Multi-mode" }, \ - { IFM_FDDI_UTP, "UTP" }, \ - { 0, NULL }, \ -} - -#define IFM_SUBTYPE_FDDI_ALIASES { \ - { IFM_FDDI_SMF, "SMF" }, \ - { IFM_FDDI_MMF, "MMF" }, \ - { IFM_FDDI_UTP, "CDDI" }, \ - { 0, NULL }, \ -} - -#define IFM_SUBTYPE_FDDI_OPTION_DESCRIPTIONS { \ - { IFM_FDDI_DA, "Dual-attach" }, \ - { 0, NULL }, \ -} - #define IFM_SUBTYPE_IEEE80211_DESCRIPTIONS { \ { IFM_IEEE80211_FH1, "FH/1Mbps" }, \ { IFM_IEEE80211_FH2, "FH/2Mbps" }, \ @@ -797,15 +720,6 @@ struct ifmedia_baudrate { { IFM_ETHER | IFM_25G_ACC, IF_Gbps(25ULL) }, \ { IFM_ETHER | IFM_25G_AOC, IF_Gbps(25ULL) }, \ \ - { IFM_TOKEN | IFM_TOK_STP4, IF_Mbps(4) }, \ - { IFM_TOKEN | IFM_TOK_STP16, IF_Mbps(16) }, \ - { IFM_TOKEN | IFM_TOK_UTP4, IF_Mbps(4) }, \ - { IFM_TOKEN | IFM_TOK_UTP16, IF_Mbps(16) }, \ - \ - { IFM_FDDI | IFM_FDDI_SMF, IF_Mbps(100) }, \ - { IFM_FDDI | IFM_FDDI_MMF, IF_Mbps(100) }, \ - { IFM_FDDI | IFM_FDDI_UTP, IF_Mbps(100) }, \ - \ { IFM_IEEE80211 | IFM_IEEE80211_FH1, IF_Mbps(1) }, \ { IFM_IEEE80211 | IFM_IEEE80211_FH2, IF_Mbps(2) }, \ { IFM_IEEE80211 | IFM_IEEE80211_DS2, IF_Mbps(2) }, \ @@ -842,10 +756,6 @@ struct ifmedia_status_description { #define IFM_STATUS_DESCRIPTIONS { \ { IFM_ETHER, IFM_AVALID, IFM_ACTIVE, \ { "no carrier", "active" } }, \ - { IFM_FDDI, IFM_AVALID, IFM_ACTIVE, \ - { "no ring", "inserted" } }, \ - { IFM_TOKEN, IFM_AVALID, IFM_ACTIVE, \ - { "no ring", "inserted" } }, \ { IFM_IEEE80211, IFM_AVALID, IFM_ACTIVE, \ { "no network", "active" } }, \ { IFM_ATM, IFM_AVALID, IFM_ACTIVE, \ diff --git a/freebsd/sys/net/if_spppsubr.c b/freebsd/sys/net/if_spppsubr.c index 1f85c00f..f5b78dec 100644 --- a/freebsd/sys/net/if_spppsubr.c +++ b/freebsd/sys/net/if_spppsubr.c @@ -4839,7 +4839,7 @@ sppp_get_ip_addrs(struct sppp *sp, u_long *src, u_long *dst, u_long *srcmask) */ si = NULL; if_addr_rlock(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) if (ifa->ifa_addr->sa_family == AF_INET) { si = (struct sockaddr_in *)ifa->ifa_addr; sm = (struct sockaddr_in *)ifa->ifa_netmask; @@ -4881,7 +4881,7 @@ sppp_set_ip_addr(struct sppp *sp, u_long src) */ si = NULL; if_addr_rlock(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family == AF_INET) { si = (struct sockaddr_in *)ifa->ifa_addr; if (si != NULL) { @@ -4943,7 +4943,7 @@ sppp_get_ip6_addrs(struct sppp *sp, struct in6_addr *src, struct in6_addr *dst, */ si = NULL; if_addr_rlock(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) if (ifa->ifa_addr->sa_family == AF_INET6) { si = (struct sockaddr_in6 *)ifa->ifa_addr; sm = (struct sockaddr_in6 *)ifa->ifa_netmask; @@ -4998,7 +4998,7 @@ sppp_set_ip6_addr(struct sppp *sp, const struct in6_addr *src) sin6 = NULL; if_addr_rlock(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family == AF_INET6) { sin6 = (struct sockaddr_in6 *)ifa->ifa_addr; if (sin6 && IN6_IS_ADDR_LINKLOCAL(&sin6->sin6_addr)) { diff --git a/freebsd/sys/net/if_stf.c b/freebsd/sys/net/if_stf.c index 2aab358c..1d16b2d7 100644 --- a/freebsd/sys/net/if_stf.c +++ b/freebsd/sys/net/if_stf.c @@ -278,7 +278,7 @@ static int stf_clone_destroy(struct if_clone *ifc, struct ifnet *ifp) { struct stf_softc *sc = ifp->if_softc; - int err; + int err __unused; err = encap_detach(sc->encap_cookie); KASSERT(err == 0, ("Unexpected error detaching encap_cookie")); @@ -386,7 +386,7 @@ stf_getsrcifa6(struct ifnet *ifp, struct in6_addr *addr, struct in6_addr *mask) struct in_addr in; if_addr_rlock(ifp); - TAILQ_FOREACH(ia, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ia, &ifp->if_addrhead, ifa_link) { if (ia->ifa_addr->sa_family != AF_INET6) continue; sin6 = (struct sockaddr_in6 *)ia->ifa_addr; @@ -563,7 +563,7 @@ stf_checkaddr4(struct stf_softc *sc, struct in_addr *in, struct ifnet *inifp) * reject packets with broadcast */ IN_IFADDR_RLOCK(&in_ifa_tracker); - TAILQ_FOREACH(ia4, &V_in_ifaddrhead, ia_link) { + CK_STAILQ_FOREACH(ia4, &V_in_ifaddrhead, ia_link) { if ((ia4->ia_ifa.ifa_ifp->if_flags & IFF_BROADCAST) == 0) continue; if (in->s_addr == ia4->ia_broadaddr.sin_addr.s_addr) { diff --git a/freebsd/sys/net/if_tap.c b/freebsd/sys/net/if_tap.c index ce143f9f..c918a14e 100644 --- a/freebsd/sys/net/if_tap.c +++ b/freebsd/sys/net/if_tap.c @@ -39,7 +39,6 @@ * $Id: if_tap.c,v 0.21 2000/07/23 21:46:02 max Exp $ */ -#include <rtems/bsd/local/opt_compat.h> #include <rtems/bsd/local/opt_inet.h> #include <sys/param.h> @@ -80,7 +79,6 @@ #include <net/if_tapvar.h> #include <net/if_tap.h> - #define CDEV_NAME "tap" #define TAPDEBUG if (tapdebug) printf @@ -551,7 +549,7 @@ tapclose(struct cdev *dev, int foo, int bar, struct thread *td) if (ifp->if_drv_flags & IFF_DRV_RUNNING) { ifp->if_drv_flags &= ~IFF_DRV_RUNNING; mtx_unlock(&tp->tap_mtx); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { rtinit(ifa, (int)RTM_DELETE, 0); } if_purgeaddrs(ifp); diff --git a/freebsd/sys/net/if_tun.c b/freebsd/sys/net/if_tun.c index 598a5c14..e4a0b02f 100644 --- a/freebsd/sys/net/if_tun.c +++ b/freebsd/sys/net/if_tun.c @@ -476,7 +476,7 @@ tunclose(struct cdev *dev, int foo, int bar, struct thread *td) ifp->if_drv_flags &= ~IFF_DRV_RUNNING; mtx_unlock(&tp->tun_mtx); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { /* deal w/IPv4 PtP destination; unlocked read */ if (ifa->ifa_addr->sa_family == AF_INET) { rtinit(ifa, (int)RTM_DELETE, @@ -518,7 +518,7 @@ tuninit(struct ifnet *ifp) #ifdef INET if_addr_rlock(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family == AF_INET) { struct sockaddr_in *si; diff --git a/freebsd/sys/net/if_var.h b/freebsd/sys/net/if_var.h index a131c496..f8e18f7e 100644 --- a/freebsd/sys/net/if_var.h +++ b/freebsd/sys/net/if_var.h @@ -70,27 +70,29 @@ struct route; /* if_output */ struct vnet; struct ifmedia; struct netmap_adapter; +struct netdump_methods; #ifdef _KERNEL #include <sys/mbuf.h> /* ifqueue only? */ #include <sys/buf_ring.h> #include <net/vnet.h> #endif /* _KERNEL */ +#include <sys/ck.h> #include <sys/counter.h> +#include <sys/epoch.h> #include <sys/lock.h> /* XXX */ #include <sys/mutex.h> /* struct ifqueue */ #include <sys/rwlock.h> /* XXX */ #include <sys/sx.h> /* XXX */ #include <sys/_task.h> /* if_link_task */ - #define IF_DUNIT_NONE -1 #include <net/altq/if_altq.h> -TAILQ_HEAD(ifnethead, ifnet); /* we use TAILQs so that the order of */ -TAILQ_HEAD(ifaddrhead, ifaddr); /* instantiation is preserved in the list */ -TAILQ_HEAD(ifmultihead, ifmultiaddr); -TAILQ_HEAD(ifgrouphead, ifg_group); +CK_STAILQ_HEAD(ifnethead, ifnet); /* we use TAILQs so that the order of */ +CK_STAILQ_HEAD(ifaddrhead, ifaddr); /* instantiation is preserved in the list */ +CK_STAILQ_HEAD(ifmultihead, ifmultiaddr); +CK_STAILQ_HEAD(ifgrouphead, ifg_group); #ifdef _KERNEL VNET_DECLARE(struct pfil_head, link_pfil_hook); /* packet filter hooks */ @@ -103,6 +105,15 @@ VNET_DECLARE(struct hhook_head *, ipsec_hhh_in[HHOOK_IPSEC_COUNT]); VNET_DECLARE(struct hhook_head *, ipsec_hhh_out[HHOOK_IPSEC_COUNT]); #define V_ipsec_hhh_in VNET(ipsec_hhh_in) #define V_ipsec_hhh_out VNET(ipsec_hhh_out) +#ifndef __rtems__ +extern epoch_t net_epoch_preempt; +extern epoch_t net_epoch; +#else /* __rtems__ */ +extern struct epoch _bsd_net_epoch_preempt; +#define net_epoch_preempt &_bsd_net_epoch_preempt +extern struct epoch _bsd_net_epoch; +#define net_epoch &_bsd_net_epoch +#endif /* __rtems__ */ #endif /* _KERNEL */ typedef enum { @@ -234,9 +245,9 @@ typedef void (if_snd_tag_free_t)(struct m_snd_tag *); */ struct ifnet { /* General book keeping of interface lists. */ - TAILQ_ENTRY(ifnet) if_link; /* all struct ifnets are chained */ + CK_STAILQ_ENTRY(ifnet) if_link; /* all struct ifnets are chained (CK_) */ LIST_ENTRY(ifnet) if_clones; /* interfaces of a cloner */ - TAILQ_HEAD(, ifg_list) if_groups; /* linked list of groups per if */ + CK_STAILQ_HEAD(, ifg_list) if_groups; /* linked list of groups per if (CK_) */ /* protected by if_addr_lock */ u_char if_alloctype; /* if_type at time of allocation */ @@ -276,7 +287,7 @@ struct ifnet { struct task if_linktask; /* task for link change events */ /* Addresses of different protocol families assigned to this if. */ - struct rwlock if_addr_lock; /* lock to protect address lists */ + struct mtx if_addr_lock; /* lock to protect address lists */ /* * if_addrhead is the list of all addresses associated to * an interface. @@ -293,7 +304,7 @@ struct ifnet { struct ifaddr *if_addr; /* pointer to link-level address */ void *if_hw_addr; /* hardware link-level address */ const u_int8_t *if_broadcastaddr; /* linklevel broadcast bytestring */ - struct rwlock if_afdata_lock; + struct mtx if_afdata_lock; void *if_afdata[AF_MAX]; int if_afdata_initialized; @@ -317,6 +328,10 @@ struct ifnet { struct route *); void (*if_input) /* input routine (from h/w driver) */ (struct ifnet *, struct mbuf *); + struct mbuf *(*if_bridge_input)(struct ifnet *, struct mbuf *); + int (*if_bridge_output)(struct ifnet *, struct mbuf *, struct sockaddr *, + struct rtentry *); + void (*if_bridge_linkstate)(struct ifnet *ifp); if_start_fn_t if_start; /* initiate output routine */ if_ioctl_fn_t if_ioctl; /* ioctl routine */ if_init_fn_t if_init; /* Init routine */ @@ -370,6 +385,14 @@ struct ifnet { #ifndef __rtems__ /* + * Netdump hooks to be called while dumping. + */ + struct netdump_methods *if_netdump_methods; +#endif /* __rtems__ */ + struct epoch_context if_epoch_ctx; + +#ifndef __rtems__ + /* * Spare fields to be added before branching a stable branch, so * that structure can be enhanced without changing the kernel * binary interface. @@ -396,14 +419,18 @@ struct rtems_ifinputreq { /* * Locks for address lists on the network interface. */ -#define IF_ADDR_LOCK_INIT(if) rw_init(&(if)->if_addr_lock, "if_addr_lock") -#define IF_ADDR_LOCK_DESTROY(if) rw_destroy(&(if)->if_addr_lock) -#define IF_ADDR_WLOCK(if) rw_wlock(&(if)->if_addr_lock) -#define IF_ADDR_WUNLOCK(if) rw_wunlock(&(if)->if_addr_lock) -#define IF_ADDR_RLOCK(if) rw_rlock(&(if)->if_addr_lock) -#define IF_ADDR_RUNLOCK(if) rw_runlock(&(if)->if_addr_lock) -#define IF_ADDR_LOCK_ASSERT(if) rw_assert(&(if)->if_addr_lock, RA_LOCKED) -#define IF_ADDR_WLOCK_ASSERT(if) rw_assert(&(if)->if_addr_lock, RA_WLOCKED) +#define IF_ADDR_LOCK_INIT(if) mtx_init(&(if)->if_addr_lock, "if_addr_lock", NULL, MTX_DEF) +#define IF_ADDR_LOCK_DESTROY(if) mtx_destroy(&(if)->if_addr_lock) +#define IF_ADDR_RLOCK(if) epoch_enter_preempt(net_epoch_preempt); +#define IF_ADDR_RUNLOCK(if) epoch_exit_preempt(net_epoch_preempt); + +#define IF_ADDR_WLOCK(if) mtx_lock(&(if)->if_addr_lock) +#define IF_ADDR_WUNLOCK(if) mtx_unlock(&(if)->if_addr_lock) +#define IF_ADDR_LOCK_ASSERT(if) MPASS(in_epoch() || mtx_owned(&(if)->if_addr_lock)) +#define IF_ADDR_WLOCK_ASSERT(if) mtx_assert(&(if)->if_addr_lock, MA_OWNED) +#define NET_EPOCH_ENTER() epoch_enter_preempt(net_epoch_preempt) +#define NET_EPOCH_EXIT() epoch_exit_preempt(net_epoch_preempt) + /* * Function variations on locking macros intended to be used by loadable @@ -435,6 +462,8 @@ EVENTHANDLER_DECLARE(ifnet_link_event, ifnet_link_event_handler_t); /* Interface up/down event */ #define IFNET_EVENT_UP 0 #define IFNET_EVENT_DOWN 1 +#define IFNET_EVENT_PCP 2 /* priority code point, PCP */ + typedef void (*ifnet_event_fn)(void *, struct ifnet *ifp, int event); EVENTHANDLER_DECLARE(ifnet_event, ifnet_event_fn); #endif /* _SYS_EVENTHANDLER_H_ */ @@ -446,18 +475,18 @@ struct ifg_group { char ifg_group[IFNAMSIZ]; u_int ifg_refcnt; void *ifg_pf_kif; - TAILQ_HEAD(, ifg_member) ifg_members; - TAILQ_ENTRY(ifg_group) ifg_next; + CK_STAILQ_HEAD(, ifg_member) ifg_members; /* (CK_) */ + CK_STAILQ_ENTRY(ifg_group) ifg_next; /* (CK_) */ }; struct ifg_member { - TAILQ_ENTRY(ifg_member) ifgm_next; + CK_STAILQ_ENTRY(ifg_member) ifgm_next; /* (CK_) */ struct ifnet *ifgm_ifp; }; struct ifg_list { struct ifg_group *ifgl_group; - TAILQ_ENTRY(ifg_list) ifgl_next; + CK_STAILQ_ENTRY(ifg_list) ifgl_next; /* (CK_) */ }; #ifdef _SYS_EVENTHANDLER_H_ @@ -473,21 +502,21 @@ EVENTHANDLER_DECLARE(group_change_event, group_change_event_handler_t); #endif /* _SYS_EVENTHANDLER_H_ */ #define IF_AFDATA_LOCK_INIT(ifp) \ - rw_init(&(ifp)->if_afdata_lock, "if_afdata") + mtx_init(&(ifp)->if_afdata_lock, "if_afdata", NULL, MTX_DEF) -#define IF_AFDATA_WLOCK(ifp) rw_wlock(&(ifp)->if_afdata_lock) -#define IF_AFDATA_RLOCK(ifp) rw_rlock(&(ifp)->if_afdata_lock) -#define IF_AFDATA_WUNLOCK(ifp) rw_wunlock(&(ifp)->if_afdata_lock) -#define IF_AFDATA_RUNLOCK(ifp) rw_runlock(&(ifp)->if_afdata_lock) +#define IF_AFDATA_WLOCK(ifp) mtx_lock(&(ifp)->if_afdata_lock) +#define IF_AFDATA_RLOCK(ifp) epoch_enter_preempt(net_epoch_preempt) +#define IF_AFDATA_WUNLOCK(ifp) mtx_unlock(&(ifp)->if_afdata_lock) +#define IF_AFDATA_RUNLOCK(ifp) epoch_exit_preempt(net_epoch_preempt) #define IF_AFDATA_LOCK(ifp) IF_AFDATA_WLOCK(ifp) #define IF_AFDATA_UNLOCK(ifp) IF_AFDATA_WUNLOCK(ifp) -#define IF_AFDATA_TRYLOCK(ifp) rw_try_wlock(&(ifp)->if_afdata_lock) -#define IF_AFDATA_DESTROY(ifp) rw_destroy(&(ifp)->if_afdata_lock) +#define IF_AFDATA_TRYLOCK(ifp) mtx_trylock(&(ifp)->if_afdata_lock) +#define IF_AFDATA_DESTROY(ifp) mtx_destroy(&(ifp)->if_afdata_lock) -#define IF_AFDATA_LOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_LOCKED) -#define IF_AFDATA_RLOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_RLOCKED) -#define IF_AFDATA_WLOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_WLOCKED) -#define IF_AFDATA_UNLOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_UNLOCKED) +#define IF_AFDATA_LOCK_ASSERT(ifp) MPASS(in_epoch() || mtx_owned(&(ifp)->if_afdata_lock)) +#define IF_AFDATA_RLOCK_ASSERT(ifp) MPASS(in_epoch()); +#define IF_AFDATA_WLOCK_ASSERT(ifp) mtx_assert(&(ifp)->if_afdata_lock, MA_OWNED) +#define IF_AFDATA_UNLOCK_ASSERT(ifp) mtx_assert(&(ifp)->if_afdata_lock, MA_NOTOWNED) /* * 72 was chosen below because it is the size of a TCP/IP @@ -515,7 +544,7 @@ struct ifaddr { struct sockaddr *ifa_netmask; /* used to determine subnet */ struct ifnet *ifa_ifp; /* back-pointer to interface */ struct carp_softc *ifa_carp; /* pointer to CARP data */ - TAILQ_ENTRY(ifaddr) ifa_link; /* queue macro glue */ + CK_STAILQ_ENTRY(ifaddr) ifa_link; /* queue macro glue */ void (*ifa_rtrequest) /* check or clean routes (+ or -)'d */ (int, struct rtentry *, struct rt_addrinfo *); u_short ifa_flags; /* mostly rt_flags for cloning */ @@ -527,6 +556,7 @@ struct ifaddr { counter_u64_t ifa_opackets; counter_u64_t ifa_ibytes; counter_u64_t ifa_obytes; + struct epoch_context ifa_epoch_ctx; }; struct ifaddr * ifa_alloc(size_t size, int flags); @@ -538,13 +568,14 @@ void ifa_ref(struct ifaddr *ifa); * structure except that it keeps track of multicast addresses. */ struct ifmultiaddr { - TAILQ_ENTRY(ifmultiaddr) ifma_link; /* queue macro glue */ + CK_STAILQ_ENTRY(ifmultiaddr) ifma_link; /* queue macro glue */ struct sockaddr *ifma_addr; /* address this membership is for */ struct sockaddr *ifma_lladdr; /* link-layer translation, if any */ struct ifnet *ifma_ifp; /* back-pointer to interface */ u_int ifma_refcount; /* reference count */ void *ifma_protospec; /* protocol-specific state, if any */ struct ifmultiaddr *ifma_llifma; /* pointer to ifma for ifma_lladdr */ + struct epoch_context ifma_epoch_ctx; }; extern struct rwlock ifnet_rwlock; @@ -565,16 +596,16 @@ extern struct sx ifnet_sxlock; * write, but also whether it was acquired with sleep support or not. */ #define IFNET_RLOCK_ASSERT() sx_assert(&ifnet_sxlock, SA_SLOCKED) -#define IFNET_RLOCK_NOSLEEP_ASSERT() rw_assert(&ifnet_rwlock, RA_RLOCKED) +#define IFNET_RLOCK_NOSLEEP_ASSERT() MPASS(in_epoch()) #define IFNET_WLOCK_ASSERT() do { \ sx_assert(&ifnet_sxlock, SA_XLOCKED); \ rw_assert(&ifnet_rwlock, RA_WLOCKED); \ } while (0) #define IFNET_RLOCK() sx_slock(&ifnet_sxlock) -#define IFNET_RLOCK_NOSLEEP() rw_rlock(&ifnet_rwlock) +#define IFNET_RLOCK_NOSLEEP() epoch_enter_preempt(net_epoch_preempt) #define IFNET_RUNLOCK() sx_sunlock(&ifnet_sxlock) -#define IFNET_RUNLOCK_NOSLEEP() rw_runlock(&ifnet_rwlock) +#define IFNET_RUNLOCK_NOSLEEP() epoch_exit_preempt(net_epoch_preempt) /* * Look up an ifnet given its index; the _ref variant also acquires a @@ -602,6 +633,12 @@ VNET_DECLARE(struct ifnet *, loif); /* first loopback interface */ #define V_if_index VNET(if_index) #define V_loif VNET(loif) +#ifdef MCAST_VERBOSE +#define MCDPRINTF printf +#else +#define MCDPRINTF(...) +#endif + int if_addgroup(struct ifnet *, const char *); int if_delgroup(struct ifnet *, const char *); int if_addmulti(struct ifnet *, struct sockaddr *, struct ifmultiaddr **); @@ -611,12 +648,14 @@ void if_attach(struct ifnet *); void if_dead(struct ifnet *); int if_delmulti(struct ifnet *, struct sockaddr *); void if_delmulti_ifma(struct ifmultiaddr *); +void if_delmulti_ifma_flags(struct ifmultiaddr *, int flags); void if_detach(struct ifnet *); void if_purgeaddrs(struct ifnet *); void if_delallmulti(struct ifnet *); void if_down(struct ifnet *); struct ifmultiaddr * if_findmulti(struct ifnet *, const struct sockaddr *); +void if_freemulti(struct ifmultiaddr *ifma); void if_free(struct ifnet *); void if_initname(struct ifnet *, const char *, int); void if_link_state_change(struct ifnet *, int); diff --git a/freebsd/sys/net/if_vlan.c b/freebsd/sys/net/if_vlan.c index 79294427..26f6bbde 100644 --- a/freebsd/sys/net/if_vlan.c +++ b/freebsd/sys/net/if_vlan.c @@ -611,7 +611,7 @@ vlan_setmulti(struct ifnet *ifp) /* Now program new ones. */ IF_ADDR_WLOCK(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; mc = malloc(sizeof(struct vlan_mc_entry), M_VLAN, M_NOWAIT); @@ -1949,6 +1949,8 @@ vlan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) } ifv->ifv_pcp = ifr->ifr_vlan_pcp; vlan_tag_recalculate(ifv); + /* broadcast event about PCP change */ + EVENTHANDLER_INVOKE(ifnet_event, ifp, IFNET_EVENT_PCP); break; case SIOCSIFCAP: diff --git a/freebsd/sys/net/iflib.h b/freebsd/sys/net/iflib.h index 02319322..140c488b 100644 --- a/freebsd/sys/net/iflib.h +++ b/freebsd/sys/net/iflib.h @@ -36,6 +36,8 @@ #include <sys/nv.h> #include <sys/gtaskqueue.h> +struct if_clone; + /* * The value type for indexing, limits max descriptors * to 65535 can be conditionally redefined to uint32_t @@ -57,6 +59,8 @@ struct if_shared_ctx; typedef struct if_shared_ctx *if_shared_ctx_t; struct if_int_delay_info; typedef struct if_int_delay_info *if_int_delay_info_t; +struct if_pseudo; +typedef struct if_pseudo *if_pseudo_t; /* * File organization: @@ -194,6 +198,9 @@ typedef struct if_softc_ctx { int isc_vectors; int isc_nrxqsets; int isc_ntxqsets; + uint8_t isc_min_tx_latency; /* disable doorbell update batching */ + uint8_t isc_rx_mvec_enable; /* generate mvecs on rx */ + uint32_t isc_txrx_budget_bytes_max; int isc_msix_bar; /* can be model specific - initialize in attach_pre */ int isc_tx_nsegments; /* can be model specific - initialize in attach_pre */ int isc_ntxd[8]; @@ -214,6 +221,7 @@ typedef struct if_softc_ctx { int isc_rss_table_mask; int isc_nrxqsets_max; int isc_ntxqsets_max; + uint32_t isc_tx_qdepth; iflib_intr_mode_t isc_intr; uint16_t isc_max_frame_size; /* set at init time by driver */ @@ -259,6 +267,7 @@ struct if_shared_ctx { int isc_rx_process_limit; int isc_tx_reclaim_thresh; int isc_flags; + const char *isc_name; }; typedef struct iflib_dma_info { @@ -320,7 +329,39 @@ typedef enum { * Driver needs frames padded to some minimum length */ #define IFLIB_NEED_ETHER_PAD 0x100 - +/* + * Packets can be freed immediately after encap + */ +#define IFLIB_TXD_ENCAP_PIO 0x00200 +/* + * Use RX completion handler + */ +#define IFLIB_RX_COMPLETION 0x00400 +/* + * Skip refilling cluster free lists + */ +#define IFLIB_SKIP_CLREFILL 0x00800 +/* + * Don't reset on hang + */ +#define IFLIB_NO_HANG_RESET 0x01000 +/* + * Don't need/want most of the niceties of + * queue management + */ +#define IFLIB_PSEUDO 0x02000 +/* + * No DMA support needed / wanted + */ +#define IFLIB_VIRTUAL 0x04000 +/* + * autogenerate a MAC address + */ +#define IFLIB_GEN_MAC 0x08000 +/* + * Interface needs admin task to ignore interface up/down status + */ +#define IFLIB_ADMIN_ALWAYS_RUN 0x10000 /* @@ -363,18 +404,18 @@ int iflib_device_deregister(if_ctx_t); -int iflib_irq_alloc(if_ctx_t, if_irq_t, int, driver_filter_t, void *filter_arg, driver_intr_t, void *arg, char *name); +int iflib_irq_alloc(if_ctx_t, if_irq_t, int, driver_filter_t, void *filter_arg, driver_intr_t, void *arg, const char *name); int iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid, - iflib_intr_type_t type, driver_filter_t *filter, - void *filter_arg, int qid, char *name); -void iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, void *arg, int qid, char *name); + iflib_intr_type_t type, driver_filter_t *filter, + void *filter_arg, int qid, const char *name); +void iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, void *arg, int qid, const char *name); void iflib_irq_free(if_ctx_t ctx, if_irq_t irq); void iflib_io_tqg_attach(struct grouptask *gt, void *uniq, int cpu, char *name); -void iflib_config_gtask_init(if_ctx_t ctx, struct grouptask *gtask, - gtask_fn_t *fn, char *name); +void iflib_config_gtask_init(void *ctx, struct grouptask *gtask, + gtask_fn_t *fn, const char *name); void iflib_config_gtask_deinit(struct grouptask *gtask); @@ -396,7 +437,7 @@ int iflib_dma_alloc_multi(if_ctx_t ctx, int *sizes, iflib_dma_info_t *dmalist, i void iflib_dma_free_multi(iflib_dma_info_t *dmalist, int count); -struct mtx *iflib_ctx_lock_get(if_ctx_t); +struct sx *iflib_ctx_lock_get(if_ctx_t); struct mtx *iflib_qset_lock_get(if_ctx_t, uint16_t); void iflib_led_create(if_ctx_t ctx); @@ -404,4 +445,9 @@ void iflib_led_create(if_ctx_t ctx); void iflib_add_int_delay_sysctl(if_ctx_t, const char *, const char *, if_int_delay_info_t, int, int); +/* + * Pseudo device support + */ +if_pseudo_t iflib_clone_register(if_shared_ctx_t); +void iflib_clone_deregister(if_pseudo_t); #endif /* __IFLIB_H_ */ diff --git a/freebsd/sys/net/pfvar.h b/freebsd/sys/net/pfvar.h index dfff0b87..824b8ec3 100644 --- a/freebsd/sys/net/pfvar.h +++ b/freebsd/sys/net/pfvar.h @@ -38,8 +38,11 @@ #include <sys/param.h> #include <sys/queue.h> #include <sys/counter.h> +#include <sys/cpuset.h> #include <sys/malloc.h> #include <sys/refcount.h> +#include <sys/lock.h> +#include <sys/rmlock.h> #include <sys/tree.h> #include <vm/uma.h> @@ -147,14 +150,15 @@ extern struct mtx pf_unlnkdrules_mtx; #define PF_UNLNKDRULES_LOCK() mtx_lock(&pf_unlnkdrules_mtx) #define PF_UNLNKDRULES_UNLOCK() mtx_unlock(&pf_unlnkdrules_mtx) -extern struct rwlock pf_rules_lock; -#define PF_RULES_RLOCK() rw_rlock(&pf_rules_lock) -#define PF_RULES_RUNLOCK() rw_runlock(&pf_rules_lock) -#define PF_RULES_WLOCK() rw_wlock(&pf_rules_lock) -#define PF_RULES_WUNLOCK() rw_wunlock(&pf_rules_lock) -#define PF_RULES_ASSERT() rw_assert(&pf_rules_lock, RA_LOCKED) -#define PF_RULES_RASSERT() rw_assert(&pf_rules_lock, RA_RLOCKED) -#define PF_RULES_WASSERT() rw_assert(&pf_rules_lock, RA_WLOCKED) +extern struct rmlock pf_rules_lock; +#define PF_RULES_RLOCK_TRACKER struct rm_priotracker _pf_rules_tracker +#define PF_RULES_RLOCK() rm_rlock(&pf_rules_lock, &_pf_rules_tracker) +#define PF_RULES_RUNLOCK() rm_runlock(&pf_rules_lock, &_pf_rules_tracker) +#define PF_RULES_WLOCK() rm_wlock(&pf_rules_lock) +#define PF_RULES_WUNLOCK() rm_wunlock(&pf_rules_lock) +#define PF_RULES_ASSERT() rm_assert(&pf_rules_lock, RA_LOCKED) +#define PF_RULES_RASSERT() rm_assert(&pf_rules_lock, RA_RLOCKED) +#define PF_RULES_WASSERT() rm_assert(&pf_rules_lock, RA_WLOCKED) extern struct sx pf_end_lock; @@ -1638,6 +1642,7 @@ void pfr_detach_table(struct pfr_ktable *); int pfr_clr_tables(struct pfr_table *, int *, int); int pfr_add_tables(struct pfr_table *, int, int *, int); int pfr_del_tables(struct pfr_table *, int, int *, int); +int pfr_table_count(struct pfr_table *, int); int pfr_get_tables(struct pfr_table *, struct pfr_table *, int *, int); int pfr_get_tstats(struct pfr_table *, struct pfr_tstats *, int *, int); int pfr_clr_tstats(struct pfr_table *, int, int *, int); diff --git a/freebsd/sys/net/route.c b/freebsd/sys/net/route.c index ade738a2..c2348e31 100644 --- a/freebsd/sys/net/route.c +++ b/freebsd/sys/net/route.c @@ -238,7 +238,7 @@ route_init(void) if (rt_numfibs == 0) rt_numfibs = 1; } -SYSINIT(route_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, route_init, 0); +SYSINIT(route_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, route_init, NULL); static int rtentry_zinit(void *mem, int size, int how) @@ -626,12 +626,12 @@ rtredirect_fib(struct sockaddr *dst, struct rib_head *rnh; ifa = NULL; + NET_EPOCH_ENTER(); rnh = rt_tables_get_rnh(fibnum, dst->sa_family); if (rnh == NULL) { error = EAFNOSUPPORT; goto out; } - /* verify the gateway is directly reachable */ if ((ifa = ifa_ifwithnet(gateway, 0, fibnum)) == NULL) { error = ENETUNREACH; @@ -685,6 +685,7 @@ rtredirect_fib(struct sockaddr *dst, info.rti_info[RTAX_DST] = dst; info.rti_info[RTAX_GATEWAY] = gateway; info.rti_info[RTAX_NETMASK] = netmask; + ifa_ref(ifa); info.rti_ifa = ifa; info.rti_flags = flags; error = rtrequest1_fib(RTM_ADD, &info, &rt, fibnum); @@ -719,7 +720,8 @@ rtredirect_fib(struct sockaddr *dst, done: if (rt) RTFREE_LOCKED(rt); -out: + out: + NET_EPOCH_EXIT(); if (error) V_rtstat.rts_badredirect++; else if (stat != NULL) @@ -730,8 +732,6 @@ out: info.rti_info[RTAX_NETMASK] = netmask; info.rti_info[RTAX_AUTHOR] = src; rt_missmsg_fib(RTM_REDIRECT, &info, flags, error, fibnum); - if (ifa != NULL) - ifa_free(ifa); } /* @@ -762,6 +762,7 @@ ifa_ifwithroute(int flags, const struct sockaddr *dst, struct sockaddr *gateway, struct ifaddr *ifa; int not_found = 0; + MPASS(in_epoch()); if ((flags & RTF_GATEWAY) == 0) { /* * If we are adding a route to an interface, @@ -790,7 +791,7 @@ ifa_ifwithroute(int flags, const struct sockaddr *dst, struct sockaddr *gateway, rt = rtalloc1_fib(gateway, 0, flags, fibnum); if (rt == NULL) - return (NULL); + goto out; /* * dismiss a gateway that is reachable only * through the default router @@ -809,21 +810,19 @@ ifa_ifwithroute(int flags, const struct sockaddr *dst, struct sockaddr *gateway, } if (!not_found && rt->rt_ifa != NULL) { ifa = rt->rt_ifa; - ifa_ref(ifa); } RT_REMREF(rt); RT_UNLOCK(rt); if (not_found || ifa == NULL) - return (NULL); + goto out; } if (ifa->ifa_addr->sa_family != dst->sa_family) { struct ifaddr *oifa = ifa; ifa = ifaof_ifpforaddr(dst, ifa->ifa_ifp); if (ifa == NULL) ifa = oifa; - else - ifa_free(oifa); } + out: return (ifa); } @@ -933,7 +932,7 @@ rt_exportinfo(struct rtentry *rt, struct rt_addrinfo *info, int flags) info->rti_flags = rt->rt_flags; info->rti_ifp = rt->rt_ifp; info->rti_ifa = rt->rt_ifa; - + ifa_ref(info->rti_ifa); if (flags & NHR_REF) { /* Do 'traditional' refcouting */ if_ref(info->rti_ifp); @@ -1309,17 +1308,19 @@ int rt_getifa_fib(struct rt_addrinfo *info, u_int fibnum) { struct ifaddr *ifa; - int error = 0; + int needref, error; /* * ifp may be specified by sockaddr_dl * when protocol address is ambiguous. */ + error = 0; + needref = (info->rti_ifa == NULL); + NET_EPOCH_ENTER(); if (info->rti_ifp == NULL && ifpaddr != NULL && ifpaddr->sa_family == AF_LINK && (ifa = ifa_ifwithnet(ifpaddr, 0, fibnum)) != NULL) { info->rti_ifp = ifa->ifa_ifp; - ifa_free(ifa); } if (info->rti_ifa == NULL && ifaaddr != NULL) info->rti_ifa = ifa_ifwithaddr(ifaaddr); @@ -1337,11 +1338,13 @@ rt_getifa_fib(struct rt_addrinfo *info, u_int fibnum) info->rti_ifa = ifa_ifwithroute(flags, sa, sa, fibnum); } - if ((ifa = info->rti_ifa) != NULL) { + if (needref && info->rti_ifa != NULL) { if (info->rti_ifp == NULL) - info->rti_ifp = ifa->ifa_ifp; + info->rti_ifp = info->rti_ifa->ifa_ifp; + ifa_ref(info->rti_ifa); } else error = ENETUNREACH; + NET_EPOCH_EXIT(); return (error); } @@ -1618,12 +1621,9 @@ rtrequest1_fib(int req, struct rt_addrinfo *info, struct rtentry **ret_nrt, error = rt_getifa_fib(info, fibnum); if (error) return (error); - } else - ifa_ref(info->rti_ifa); - ifa = info->rti_ifa; + } rt = uma_zalloc(V_rtzone, M_NOWAIT); if (rt == NULL) { - ifa_free(ifa); return (ENOBUFS); } rt->rt_flags = RTF_UP | flags; @@ -1632,7 +1632,6 @@ rtrequest1_fib(int req, struct rt_addrinfo *info, struct rtentry **ret_nrt, * Add the gateway. Possibly re-malloc-ing the storage for it. */ if ((error = rt_setgate(rt, dst, gateway)) != 0) { - ifa_free(ifa); uma_zfree(V_rtzone, rt); return (error); } @@ -1655,6 +1654,8 @@ rtrequest1_fib(int req, struct rt_addrinfo *info, struct rtentry **ret_nrt, * This moved from below so that rnh->rnh_addaddr() can * examine the ifa and ifa->ifa_ifp if it so desires. */ + ifa = info->rti_ifa; + ifa_ref(ifa); rt->rt_ifa = ifa; rt->rt_ifp = ifa->ifa_ifp; rt->rt_weight = 1; @@ -1819,6 +1820,7 @@ rtrequest1_fib_change(struct rib_head *rnh, struct rt_addrinfo *info, if (rt->rt_ifa->ifa_rtrequest != NULL) rt->rt_ifa->ifa_rtrequest(RTM_DELETE, rt, info); ifa_free(rt->rt_ifa); + rt->rt_ifa = NULL; } /* Update gateway address */ if (info->rti_info[RTAX_GATEWAY] != NULL) { @@ -1870,8 +1872,10 @@ rtrequest1_fib_change(struct rib_head *rnh, struct rt_addrinfo *info, } bad: RT_UNLOCK(rt); - if (free_ifa != 0) + if (free_ifa != 0) { ifa_free(info->rti_ifa); + info->rti_ifa = NULL; + } return (error); } @@ -2090,6 +2094,7 @@ rtinit1(struct ifaddr *ifa, int cmd, int flags, int fibnum) * Do the actual request */ bzero((caddr_t)&info, sizeof(info)); + ifa_ref(ifa); info.rti_ifa = ifa; info.rti_flags = flags | (ifa->ifa_flags & ~IFA_RTSELF) | RTF_PINNED; diff --git a/freebsd/sys/net/rtsock.c b/freebsd/sys/net/rtsock.c index 900413a0..c0c5c5c2 100644 --- a/freebsd/sys/net/rtsock.c +++ b/freebsd/sys/net/rtsock.c @@ -33,7 +33,6 @@ * @(#)rtsock.c 8.7 (Berkeley) 10/12/95 * $FreeBSD$ */ -#include <rtems/bsd/local/opt_compat.h> #include <rtems/bsd/local/opt_mpath.h> #include <rtems/bsd/local/opt_inet.h> #include <rtems/bsd/local/opt_inet6.h> @@ -467,7 +466,7 @@ rtm_get_jailed(struct rt_addrinfo *info, struct ifnet *ifp, * that belongs to the jail. */ IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { struct sockaddr *sa; sa = ifa->ifa_addr; if (sa->sa_family != AF_INET) @@ -509,7 +508,7 @@ rtm_get_jailed(struct rt_addrinfo *info, struct ifnet *ifp, * that belongs to the jail. */ IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { struct sockaddr *sa; sa = ifa->ifa_addr; if (sa->sa_family != AF_INET6) @@ -797,12 +796,14 @@ route_output(struct mbuf *m, struct socket *so, ...) rt->rt_ifp->if_type == IFT_PROPVIRTUAL) { struct ifaddr *ifa; + NET_EPOCH_ENTER(); ifa = ifa_ifwithnet(info.rti_info[RTAX_DST], 1, RT_ALL_FIBS); if (ifa != NULL) rt_maskedcopy(ifa->ifa_addr, &laddr, ifa->ifa_netmask); + NET_EPOCH_EXIT(); } else rt_maskedcopy(rt->rt_ifa->ifa_addr, &laddr, @@ -1424,7 +1425,10 @@ rt_newmaddrmsg(int cmd, struct ifmultiaddr *ifma) bzero((caddr_t)&info, sizeof(info)); info.rti_info[RTAX_IFA] = ifma->ifma_addr; - info.rti_info[RTAX_IFP] = ifp ? ifp->if_addr->ifa_addr : NULL; + if (ifp && ifp->if_addr) + info.rti_info[RTAX_IFP] = ifp->if_addr->ifa_addr; + else + info.rti_info[RTAX_IFP] = NULL; /* * If a link-layer address is present, present it as a ``gateway'' * (similarly to how ARP entries, e.g., are presented). @@ -1746,7 +1750,7 @@ sysctl_iflist(int af, struct walkarg *w) bzero((caddr_t)&info, sizeof(info)); bzero(&ifd, sizeof(ifd)); IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (w->w_arg && w->w_arg != ifp->if_index) continue; if_data_copy(ifp, &ifd); @@ -1767,7 +1771,7 @@ sysctl_iflist(int af, struct walkarg *w) if (error) goto done; } - while ((ifa = TAILQ_NEXT(ifa, ifa_link)) != NULL) { + while ((ifa = CK_STAILQ_NEXT(ifa, ifa_link)) != NULL) { if (af && af != ifa->ifa_addr->sa_family) continue; if (prison_if(w->w_req->td->td_ucred, @@ -1816,13 +1820,13 @@ sysctl_ifmalist(int af, struct walkarg *w) bzero((caddr_t)&info, sizeof(info)); IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (w->w_arg && w->w_arg != ifp->if_index) continue; ifa = ifp->if_addr; info.rti_info[RTAX_IFP] = ifa ? ifa->ifa_addr : NULL; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (af && af != ifma->ifma_addr->sa_family) continue; if (prison_if(w->w_req->td->td_ucred, diff --git a/freebsd/sys/netinet/cc/cc_newreno.c b/freebsd/sys/netinet/cc/cc_newreno.c index b7f59520..4d5f8644 100644 --- a/freebsd/sys/netinet/cc/cc_newreno.c +++ b/freebsd/sys/netinet/cc/cc_newreno.c @@ -83,7 +83,7 @@ static MALLOC_DEFINE(M_NEWRENO, "newreno data", #define CAST_PTR_INT(X) (*((int*)(X))) -static int newreno_cb_init(struct cc_var *ccv); +static void newreno_cb_destroy(struct cc_var *ccv); static void newreno_ack_received(struct cc_var *ccv, uint16_t type); static void newreno_after_idle(struct cc_var *ccv); static void newreno_cong_signal(struct cc_var *ccv, uint32_t type); @@ -97,7 +97,7 @@ static VNET_DEFINE(uint32_t, newreno_beta_ecn) = 80; struct cc_algo newreno_cc_algo = { .name = "newreno", - .cb_init = newreno_cb_init, + .cb_destroy = newreno_cb_destroy, .ack_received = newreno_ack_received, .after_idle = newreno_after_idle, .cong_signal = newreno_cong_signal, @@ -110,18 +110,28 @@ struct newreno { uint32_t beta_ecn; }; -int -newreno_cb_init(struct cc_var *ccv) +static inline struct newreno * +newreno_malloc(struct cc_var *ccv) { - struct newreno *nreno; + struct newreno *nreno; - nreno = malloc(sizeof(struct newreno), M_NEWRENO, M_NOWAIT|M_ZERO); + nreno = malloc(sizeof(struct newreno), M_NEWRENO, M_NOWAIT); if (nreno != NULL) { + /* NB: nreno is not zeroed, so initialise all fields. */ nreno->beta = V_newreno_beta; nreno->beta_ecn = V_newreno_beta_ecn; + ccv->cc_data = nreno; } - return (0); + return (nreno); +} + +static void +newreno_cb_destroy(struct cc_var *ccv) +{ + + if (ccv->cc_data != NULL) + free(ccv->cc_data, M_NEWRENO); } static void @@ -226,20 +236,18 @@ static void newreno_cong_signal(struct cc_var *ccv, uint32_t type) { struct newreno *nreno; - uint32_t cwin, factor; + uint32_t beta, beta_ecn, cwin, factor; u_int mss; - factor = V_newreno_beta; - nreno = ccv->cc_data; - if (nreno != NULL) { - if (V_cc_do_abe) - factor = (type == CC_ECN ? nreno->beta_ecn: nreno->beta); - else - factor = nreno->beta; - } - cwin = CCV(ccv, snd_cwnd); mss = CCV(ccv, t_maxseg); + nreno = ccv->cc_data; + beta = (nreno == NULL) ? V_newreno_beta : nreno->beta; + beta_ecn = (nreno == NULL) ? V_newreno_beta_ecn : nreno->beta_ecn; + if (V_cc_do_abe && type == CC_ECN) + factor = beta_ecn; + else + factor = beta; /* Catch algos which mistakenly leak private signal types. */ KASSERT((type & CC_SIGPRIVMASK) == 0, @@ -255,8 +263,8 @@ newreno_cong_signal(struct cc_var *ccv, uint32_t type) V_cc_do_abe && V_cc_abe_frlossreduce)) { CCV(ccv, snd_ssthresh) = ((uint64_t)CCV(ccv, snd_ssthresh) * - (uint64_t)nreno->beta) / - (100ULL * (uint64_t)nreno->beta_ecn); + (uint64_t)beta) / + (100ULL * (uint64_t)beta_ecn); } if (!IN_CONGRECOVERY(CCV(ccv, t_flags))) CCV(ccv, snd_ssthresh) = cwin; @@ -280,7 +288,6 @@ static void newreno_post_recovery(struct cc_var *ccv) { int pipe; - pipe = 0; if (IN_FASTRECOVERY(CCV(ccv, t_flags))) { /* @@ -304,7 +311,7 @@ newreno_post_recovery(struct cc_var *ccv) } } -int +static int newreno_ctl_output(struct cc_var *ccv, struct sockopt *sopt, void *buf) { struct newreno *nreno; @@ -315,9 +322,15 @@ newreno_ctl_output(struct cc_var *ccv, struct sockopt *sopt, void *buf) nreno = ccv->cc_data; opt = buf; - + switch (sopt->sopt_dir) { case SOPT_SET: + /* We cannot set without cc_data memory. */ + if (nreno == NULL) { + nreno = newreno_malloc(ccv); + if (nreno == NULL) + return (ENOMEM); + } switch (opt->name) { case CC_NEWRENO_BETA: nreno->beta = opt->val; @@ -330,17 +343,21 @@ newreno_ctl_output(struct cc_var *ccv, struct sockopt *sopt, void *buf) default: return (ENOPROTOOPT); } + break; case SOPT_GET: switch (opt->name) { case CC_NEWRENO_BETA: - opt->val = nreno->beta; + opt->val = (nreno == NULL) ? + V_newreno_beta : nreno->beta; break; case CC_NEWRENO_BETA_ECN: - opt->val = nreno->beta_ecn; + opt->val = (nreno == NULL) ? + V_newreno_beta_ecn : nreno->beta_ecn; break; default: return (ENOPROTOOPT); } + break; default: return (EINVAL); } @@ -351,6 +368,7 @@ newreno_ctl_output(struct cc_var *ccv, struct sockopt *sopt, void *buf) static int newreno_beta_handler(SYSCTL_HANDLER_ARGS) { + if (req->newptr != NULL ) { if (arg1 == &VNET_NAME(newreno_beta_ecn) && !V_cc_do_abe) return (EACCES); diff --git a/freebsd/sys/netinet/if_ether.c b/freebsd/sys/netinet/if_ether.c index 699af2e4..0d608180 100644 --- a/freebsd/sys/netinet/if_ether.c +++ b/freebsd/sys/netinet/if_ether.c @@ -364,7 +364,7 @@ arprequest(struct ifnet *ifp, const struct in_addr *sip, struct ifaddr *ifa; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; @@ -696,14 +696,6 @@ arpintr(struct mbuf *m) hlen = ETHER_ADDR_LEN; /* RFC 826 */ layer = "ethernet"; break; - case ARPHRD_IEEE802: - hlen = 6; /* RFC 1390, FDDI_ADDR_LEN */ - layer = "fddi"; - break; - case ARPHRD_ARCNET: - hlen = 1; /* RFC 1201, ARC_ADDR_LEN */ - layer = "arcnet"; - break; case ARPHRD_INFINIBAND: hlen = 20; /* RFC 4391, INFINIBAND_ALEN */ layer = "infiniband"; @@ -896,7 +888,7 @@ in_arpinput(struct mbuf *m) * as a dummy address for the rest of the function. */ IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) if (ifa->ifa_addr->sa_family == AF_INET && (ifa->ifa_carp == NULL || (*carp_iamatch_p)(ifa, &enaddr))) { @@ -911,7 +903,7 @@ in_arpinput(struct mbuf *m) * If bridging, fall back to using any inet address. */ IN_IFADDR_RLOCK(&in_ifa_tracker); - if (!bridged || (ia = TAILQ_FIRST(&V_in_ifaddrhead)) == NULL) { + if (!bridged || (ia = CK_STAILQ_FIRST(&V_in_ifaddrhead)) == NULL) { IN_IFADDR_RUNLOCK(&in_ifa_tracker); goto drop; } @@ -1455,7 +1447,7 @@ arp_handle_ifllchange(struct ifnet *ifp) { struct ifaddr *ifa; - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family == AF_INET) arp_ifinit(ifp, ifa); } diff --git a/freebsd/sys/netinet/igmp.c b/freebsd/sys/netinet/igmp.c index cf319470..a4b99f62 100644 --- a/freebsd/sys/netinet/igmp.c +++ b/freebsd/sys/netinet/igmp.c @@ -138,7 +138,7 @@ static int igmp_v3_enqueue_group_record(struct mbufq *, struct in_multi *, const int, const int, const int); static int igmp_v3_enqueue_filter_change(struct mbufq *, struct in_multi *); -static void igmp_v3_process_group_timers(struct igmp_ifsoftc *, +static void igmp_v3_process_group_timers(struct in_multi_head *, struct mbufq *, struct mbufq *, struct in_multi *, const int); static int igmp_v3_merge_state_changes(struct in_multi *, @@ -164,12 +164,12 @@ static const struct netisr_handler igmp_nh = { * themselves are not virtualized. * * Locking: - * * The permitted lock order is: IN_MULTI_LOCK, IGMP_LOCK, IF_ADDR_LOCK. + * * The permitted lock order is: IN_MULTI_LIST_LOCK, IGMP_LOCK, IF_ADDR_LOCK. * Any may be taken independently; if any are held at the same * time, the above lock order must be followed. * * All output is delegated to the netisr. * Now that Giant has been eliminated, the netisr may be inlined. - * * IN_MULTI_LOCK covers in_multi. + * * IN_MULTI_LIST_LOCK covers in_multi. * * IGMP_LOCK covers igmp_ifsoftc and any global variables in this file, * including the output queue. * * IF_ADDR_LOCK covers if_multiaddrs, which is used for a variety of @@ -443,7 +443,7 @@ sysctl_igmp_ifinfo(SYSCTL_HANDLER_ARGS) if (error) return (error); - IN_MULTI_LOCK(); + IN_MULTI_LIST_LOCK(); IGMP_LOCK(); if (name[0] <= 0 || name[0] > V_if_index) { @@ -477,7 +477,7 @@ sysctl_igmp_ifinfo(SYSCTL_HANDLER_ARGS) out_locked: IGMP_UNLOCK(); - IN_MULTI_UNLOCK(); + IN_MULTI_LIST_UNLOCK(); return (error); } @@ -588,7 +588,6 @@ igi_alloc_locked(/*const*/ struct ifnet *ifp) igi->igi_qi = IGMP_QI_INIT; igi->igi_qri = IGMP_QRI_INIT; igi->igi_uri = IGMP_URI_INIT; - SLIST_INIT(&igi->igi_relinmhead); mbufq_init(&igi->igi_gq, IGMP_MAX_RESPONSE_PACKETS); LIST_INSERT_HEAD(&V_igi_head, igi, igi_link); @@ -613,44 +612,37 @@ void igmp_ifdetach(struct ifnet *ifp) { struct igmp_ifsoftc *igi; - struct ifmultiaddr *ifma; - struct in_multi *inm, *tinm; - + struct ifmultiaddr *ifma, *next; + struct in_multi *inm; + struct in_multi_head inm_free_tmp; CTR3(KTR_IGMPV3, "%s: called for ifp %p(%s)", __func__, ifp, ifp->if_xname); + SLIST_INIT(&inm_free_tmp); IGMP_LOCK(); igi = ((struct in_ifinfo *)ifp->if_afdata[AF_INET])->ii_igmp; if (igi->igi_version == IGMP_VERSION_3) { - IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + IF_ADDR_WLOCK(ifp); + restart: + CK_STAILQ_FOREACH_SAFE(ifma, &ifp->if_multiaddrs, ifma_link, next) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) continue; -#if 0 - KASSERT(ifma->ifma_protospec != NULL, - ("%s: ifma_protospec is NULL", __func__)); -#endif inm = (struct in_multi *)ifma->ifma_protospec; - if (inm->inm_state == IGMP_LEAVING_MEMBER) { - SLIST_INSERT_HEAD(&igi->igi_relinmhead, - inm, inm_nrele); - } + if (inm->inm_state == IGMP_LEAVING_MEMBER) + inm_rele_locked(&inm_free_tmp, inm); inm_clear_recorded(inm); + if (__predict_false(ifma_restart)) { + ifma_restart = false; + goto restart; + } } - IF_ADDR_RUNLOCK(ifp); - /* - * Free the in_multi reference(s) for this IGMP lifecycle. - */ - SLIST_FOREACH_SAFE(inm, &igi->igi_relinmhead, inm_nrele, - tinm) { - SLIST_REMOVE_HEAD(&igi->igi_relinmhead, inm_nrele); - inm_release_locked(inm); - } + IF_ADDR_WUNLOCK(ifp); + inm_release_list_deferred(&inm_free_tmp); } - IGMP_UNLOCK(); + } /* @@ -686,11 +678,6 @@ igi_delete_locked(const struct ifnet *ifp) mbufq_drain(&igi->igi_gq); LIST_REMOVE(igi, igi_link); - - KASSERT(SLIST_EMPTY(&igi->igi_relinmhead), - ("%s: there are dangling in_multi references", - __func__)); - free(igi, M_IGMP); return; } @@ -724,7 +711,7 @@ igmp_input_v1_query(struct ifnet *ifp, const struct ip *ip, } IGMPSTAT_INC(igps_rcv_gen_queries); - IN_MULTI_LOCK(); + IN_MULTI_LIST_LOCK(); IGMP_LOCK(); igi = ((struct in_ifinfo *)ifp->if_afdata[AF_INET])->ii_igmp; @@ -749,7 +736,7 @@ igmp_input_v1_query(struct ifnet *ifp, const struct ip *ip, * except those which are already running. */ IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) continue; @@ -780,7 +767,7 @@ igmp_input_v1_query(struct ifnet *ifp, const struct ip *ip, out_locked: IGMP_UNLOCK(); - IN_MULTI_UNLOCK(); + IN_MULTI_LIST_UNLOCK(); return (0); } @@ -818,7 +805,7 @@ igmp_input_v2_query(struct ifnet *ifp, const struct ip *ip, IGMPSTAT_INC(igps_rcv_group_queries); } - IN_MULTI_LOCK(); + IN_MULTI_LIST_LOCK(); IGMP_LOCK(); igi = ((struct in_ifinfo *)ifp->if_afdata[AF_INET])->ii_igmp; @@ -850,7 +837,7 @@ igmp_input_v2_query(struct ifnet *ifp, const struct ip *ip, CTR2(KTR_IGMPV3, "process v2 general query on ifp %p(%s)", ifp, ifp->if_xname); IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) continue; @@ -874,7 +861,7 @@ igmp_input_v2_query(struct ifnet *ifp, const struct ip *ip, out_locked: IGMP_UNLOCK(); - IN_MULTI_UNLOCK(); + IN_MULTI_LIST_UNLOCK(); return (0); } @@ -901,7 +888,7 @@ igmp_v2_update_group(struct in_multi *inm, const int timer) CTR4(KTR_IGMPV3, "0x%08x: %s/%s timer=%d", __func__, ntohl(inm->inm_addr.s_addr), inm->inm_ifp->if_xname, timer); - IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_LOCK_ASSERT(); switch (inm->inm_state) { case IGMP_NOT_MEMBER: @@ -1013,7 +1000,7 @@ igmp_input_v3_query(struct ifnet *ifp, const struct ip *ip, IGMPSTAT_INC(igps_rcv_gsr_queries); } - IN_MULTI_LOCK(); + IN_MULTI_LIST_LOCK(); IGMP_LOCK(); igi = ((struct in_ifinfo *)ifp->if_afdata[AF_INET])->ii_igmp; @@ -1094,7 +1081,7 @@ igmp_input_v3_query(struct ifnet *ifp, const struct ip *ip, out_locked: IGMP_UNLOCK(); - IN_MULTI_UNLOCK(); + IN_MULTI_LIST_UNLOCK(); return (0); } @@ -1111,7 +1098,7 @@ igmp_input_v3_group_query(struct in_multi *inm, struct igmp_ifsoftc *igi, int retval; uint16_t nsrc; - IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_LOCK_ASSERT(); IGMP_LOCK_ASSERT(); retval = 0; @@ -1233,11 +1220,11 @@ igmp_input_v1_report(struct ifnet *ifp, /*const*/ struct ip *ip, * Replace 0.0.0.0 with the subnet address if told to do so. */ if (V_igmp_recvifkludge && in_nullhost(ip->ip_src)) { + NET_EPOCH_ENTER(); IFP_TO_IA(ifp, ia, &in_ifa_tracker); - if (ia != NULL) { + if (ia != NULL) ip->ip_src.s_addr = htonl(ia->ia_subnet); - ifa_free(&ia->ia_ifa); - } + NET_EPOCH_EXIT(); } CTR3(KTR_IGMPV3, "process v1 report 0x%08x on ifp %p(%s)", @@ -1248,7 +1235,7 @@ igmp_input_v1_report(struct ifnet *ifp, /*const*/ struct ip *ip, * If we are a member of this group, and our membership should be * reported, stop our group timer and transition to the 'lazy' state. */ - IN_MULTI_LOCK(); + IN_MULTI_LIST_LOCK(); inm = inm_lookup(ifp, igmp->igmp_group); if (inm != NULL) { struct igmp_ifsoftc *igi; @@ -1307,7 +1294,7 @@ igmp_input_v1_report(struct ifnet *ifp, /*const*/ struct ip *ip, } out_locked: - IN_MULTI_UNLOCK(); + IN_MULTI_LIST_UNLOCK(); return (0); } @@ -1330,24 +1317,23 @@ igmp_input_v2_report(struct ifnet *ifp, /*const*/ struct ip *ip, * leave requires knowing that we are the only member of a * group. */ + NET_EPOCH_ENTER(); IFP_TO_IA(ifp, ia, &in_ifa_tracker); if (ia != NULL && in_hosteq(ip->ip_src, IA_SIN(ia)->sin_addr)) { - ifa_free(&ia->ia_ifa); + NET_EPOCH_EXIT(); return (0); } IGMPSTAT_INC(igps_rcv_reports); if (ifp->if_flags & IFF_LOOPBACK) { - if (ia != NULL) - ifa_free(&ia->ia_ifa); + NET_EPOCH_EXIT(); return (0); } if (!IN_MULTICAST(ntohl(igmp->igmp_group.s_addr)) || !in_hosteq(igmp->igmp_group, ip->ip_dst)) { - if (ia != NULL) - ifa_free(&ia->ia_ifa); + NET_EPOCH_EXIT(); IGMPSTAT_INC(igps_rcv_badreports); return (EINVAL); } @@ -1363,8 +1349,7 @@ igmp_input_v2_report(struct ifnet *ifp, /*const*/ struct ip *ip, if (ia != NULL) ip->ip_src.s_addr = htonl(ia->ia_subnet); } - if (ia != NULL) - ifa_free(&ia->ia_ifa); + NET_EPOCH_EXIT(); CTR3(KTR_IGMPV3, "process v2 report 0x%08x on ifp %p(%s)", ntohl(igmp->igmp_group.s_addr), ifp, ifp->if_xname); @@ -1375,7 +1360,7 @@ igmp_input_v2_report(struct ifnet *ifp, /*const*/ struct ip *ip, * reported, and our group timer is pending or about to be reset, * stop our group timer by transitioning to the 'lazy' state. */ - IN_MULTI_LOCK(); + IN_MULTI_LIST_LOCK(); inm = inm_lookup(ifp, igmp->igmp_group); if (inm != NULL) { struct igmp_ifsoftc *igi; @@ -1420,7 +1405,7 @@ igmp_input_v2_report(struct ifnet *ifp, /*const*/ struct ip *ip, } out_locked: - IN_MULTI_UNLOCK(); + IN_MULTI_LIST_UNLOCK(); return (0); } @@ -1647,8 +1632,9 @@ igmp_fasttimo_vnet(void) struct mbufq qrq; /* Query response packets */ struct ifnet *ifp; struct igmp_ifsoftc *igi; - struct ifmultiaddr *ifma; + struct ifmultiaddr *ifma, *next; struct in_multi *inm; + struct in_multi_head inm_free_tmp; int loop, uri_fasthz; loop = 0; @@ -1664,7 +1650,8 @@ igmp_fasttimo_vnet(void) !V_state_change_timers_running) return; - IN_MULTI_LOCK(); + SLIST_INIT(&inm_free_tmp); + IN_MULTI_LIST_LOCK(); IGMP_LOCK(); /* @@ -1709,8 +1696,9 @@ igmp_fasttimo_vnet(void) mbufq_init(&scq, IGMP_MAX_STATE_CHANGE_PACKETS); } - IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + IF_ADDR_WLOCK(ifp); + restart: + CK_STAILQ_FOREACH_SAFE(ifma, &ifp->if_multiaddrs, ifma_link, next) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) continue; @@ -1722,16 +1710,18 @@ igmp_fasttimo_vnet(void) igi->igi_version); break; case IGMP_VERSION_3: - igmp_v3_process_group_timers(igi, &qrq, + igmp_v3_process_group_timers(&inm_free_tmp, &qrq, &scq, inm, uri_fasthz); break; } + if (__predict_false(ifma_restart)) { + ifma_restart = false; + goto restart; + } } - IF_ADDR_RUNLOCK(ifp); + IF_ADDR_WUNLOCK(ifp); if (igi->igi_version == IGMP_VERSION_3) { - struct in_multi *tinm; - igmp_dispatch_queue(&qrq, 0, loop); igmp_dispatch_queue(&scq, 0, loop); @@ -1739,18 +1729,13 @@ igmp_fasttimo_vnet(void) * Free the in_multi reference(s) for this * IGMP lifecycle. */ - SLIST_FOREACH_SAFE(inm, &igi->igi_relinmhead, - inm_nrele, tinm) { - SLIST_REMOVE_HEAD(&igi->igi_relinmhead, - inm_nrele); - inm_release_locked(inm); - } + inm_release_list_deferred(&inm_free_tmp); } } out_locked: IGMP_UNLOCK(); - IN_MULTI_UNLOCK(); + IN_MULTI_LIST_UNLOCK(); } /* @@ -1762,7 +1747,7 @@ igmp_v1v2_process_group_timer(struct in_multi *inm, const int version) { int report_timer_expired; - IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_LOCK_ASSERT(); IGMP_LOCK_ASSERT(); if (inm->inm_timer == 0) { @@ -1804,14 +1789,14 @@ igmp_v1v2_process_group_timer(struct in_multi *inm, const int version) * Note: Unlocked read from igi. */ static void -igmp_v3_process_group_timers(struct igmp_ifsoftc *igi, +igmp_v3_process_group_timers(struct in_multi_head *inmh, struct mbufq *qrq, struct mbufq *scq, struct in_multi *inm, const int uri_fasthz) { int query_response_timer_expired; int state_change_retransmit_timer_expired; - IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_LOCK_ASSERT(); IGMP_LOCK_ASSERT(); query_response_timer_expired = 0; @@ -1861,7 +1846,7 @@ igmp_v3_process_group_timers(struct igmp_ifsoftc *igi, * immediate transmission. */ if (query_response_timer_expired) { - int retval; + int retval __unused; retval = igmp_v3_enqueue_group_record(qrq, inm, 0, 1, (inm->inm_state == IGMP_SG_QUERY_PENDING_MEMBER)); @@ -1909,8 +1894,7 @@ igmp_v3_process_group_timers(struct igmp_ifsoftc *igi, if (inm->inm_state == IGMP_LEAVING_MEMBER && inm->inm_scrv == 0) { inm->inm_state = IGMP_NOT_MEMBER; - SLIST_INSERT_HEAD(&igi->igi_relinmhead, - inm, inm_nrele); + inm_rele_locked(inmh, inm); } } break; @@ -1931,7 +1915,7 @@ static void igmp_v3_suppress_group_record(struct in_multi *inm) { - IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_LOCK_ASSERT(); KASSERT(inm->inm_igi->igi_version == IGMP_VERSION_3, ("%s: not IGMPv3 mode on link", __func__)); @@ -2005,13 +1989,15 @@ igmp_v3_cancel_link_timers(struct igmp_ifsoftc *igi) { struct ifmultiaddr *ifma; struct ifnet *ifp; - struct in_multi *inm, *tinm; + struct in_multi *inm; + struct in_multi_head inm_free_tmp; CTR3(KTR_IGMPV3, "%s: cancel v3 timers on ifp %p(%s)", __func__, igi->igi_ifp, igi->igi_ifp->if_xname); - IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_LOCK_ASSERT(); IGMP_LOCK_ASSERT(); + SLIST_INIT(&inm_free_tmp); /* * Stop the v3 General Query Response on this link stone dead. @@ -2026,7 +2012,7 @@ igmp_v3_cancel_link_timers(struct igmp_ifsoftc *igi) */ ifp = igi->igi_ifp; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) continue; @@ -2052,7 +2038,7 @@ igmp_v3_cancel_link_timers(struct igmp_ifsoftc *igi) * message is sent upstream to the old querier -- * transition to NOT would lose the leave and race. */ - SLIST_INSERT_HEAD(&igi->igi_relinmhead, inm, inm_nrele); + inm_rele_locked(&inm_free_tmp, inm); /* FALLTHROUGH */ case IGMP_G_QUERY_PENDING_MEMBER: case IGMP_SG_QUERY_PENDING_MEMBER: @@ -2071,10 +2057,8 @@ igmp_v3_cancel_link_timers(struct igmp_ifsoftc *igi) mbufq_drain(&inm->inm_scq); } IF_ADDR_RUNLOCK(ifp); - SLIST_FOREACH_SAFE(inm, &igi->igi_relinmhead, inm_nrele, tinm) { - SLIST_REMOVE_HEAD(&igi->igi_relinmhead, inm_nrele); - inm_release_locked(inm); - } + + inm_release_list_deferred(&inm_free_tmp); } /* @@ -2201,7 +2185,7 @@ igmp_v1v2_queue_report(struct in_multi *inm, const int type) struct ip *ip; struct mbuf *m; - IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_LOCK_ASSERT(); IGMP_LOCK_ASSERT(); ifp = inm->inm_ifp; @@ -2278,10 +2262,8 @@ igmp_change_state(struct in_multi *inm) struct ifnet *ifp; int error; - IN_MULTI_LOCK_ASSERT(); - error = 0; - + IN_MULTI_LOCK_ASSERT(); /* * Try to detect if the upper layer just asked us to change state * for an interface which has now gone away. @@ -2381,9 +2363,10 @@ igmp_initial_join(struct in_multi *inm, struct igmp_ifsoftc *igi) * group around for the final INCLUDE {} enqueue. */ if (igi->igi_version == IGMP_VERSION_3 && - inm->inm_state == IGMP_LEAVING_MEMBER) - inm_release_locked(inm); - + inm->inm_state == IGMP_LEAVING_MEMBER) { + MPASS(inm->inm_refcount > 1); + inm_rele_locked(NULL, inm); + } inm->inm_state = IGMP_REPORTING_MEMBER; switch (igi->igi_version) { @@ -2475,7 +2458,7 @@ igmp_handle_state_change(struct in_multi *inm, struct igmp_ifsoftc *igi) ifp = inm->inm_ifp; - IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_LOCK_ASSERT(); IGMP_LOCK_ASSERT(); KASSERT(igi && igi->igi_ifp == ifp, ("%s: inconsistent ifp", __func__)); @@ -2533,7 +2516,7 @@ igmp_final_leave(struct in_multi *inm, struct igmp_ifsoftc *igi) __func__, ntohl(inm->inm_addr.s_addr), inm->inm_ifp, inm->inm_ifp->if_xname); - IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_LOCK_ASSERT(); IGMP_LOCK_ASSERT(); switch (inm->inm_state) { @@ -2579,7 +2562,7 @@ igmp_final_leave(struct in_multi *inm, struct igmp_ifsoftc *igi) inm->inm_state = IGMP_NOT_MEMBER; inm->inm_sctimer = 0; } else { - int retval; + int retval __unused; inm_acquire_locked(inm); @@ -2652,7 +2635,7 @@ igmp_v3_enqueue_group_record(struct mbufq *mq, struct in_multi *inm, struct ifnet *ifp; struct ip_msource *ims, *nims; struct mbuf *m0, *m, *md; - int error, is_filter_list_change; + int is_filter_list_change; int minrec0len, m0srcs, msrcs, nbytes, off; int record_has_sources; int now; @@ -2660,9 +2643,8 @@ igmp_v3_enqueue_group_record(struct mbufq *mq, struct in_multi *inm, in_addr_t naddr; uint8_t mode; - IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_LOCK_ASSERT(); - error = 0; ifp = inm->inm_ifp; is_filter_list_change = 0; m = NULL; @@ -3020,7 +3002,7 @@ igmp_v3_enqueue_filter_change(struct mbufq *mq, struct in_multi *inm) uint8_t mode, now, then; rectype_t crt, drt, nrt; - IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_LOCK_ASSERT(); if (inm->inm_nsrc == 0 || (inm->inm_st[0].iss_asm > 0 && inm->inm_st[1].iss_asm > 0)) @@ -3223,7 +3205,7 @@ igmp_v3_merge_state_changes(struct in_multi *inm, struct mbufq *scq) domerge = 0; recslen = 0; - IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_LOCK_ASSERT(); IGMP_LOCK_ASSERT(); /* @@ -3320,9 +3302,9 @@ igmp_v3_dispatch_general_query(struct igmp_ifsoftc *igi) struct ifmultiaddr *ifma; struct ifnet *ifp; struct in_multi *inm; - int retval, loop; + int retval __unused, loop; - IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_LOCK_ASSERT(); IGMP_LOCK_ASSERT(); KASSERT(igi->igi_version == IGMP_VERSION_3, @@ -3340,7 +3322,7 @@ igmp_v3_dispatch_general_query(struct igmp_ifsoftc *igi) ifp = igi->igi_ifp; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) continue; @@ -3544,11 +3526,11 @@ igmp_v3_encap_report(struct ifnet *ifp, struct mbuf *m) if (m->m_flags & M_IGMP_LOOP) { struct in_ifaddr *ia; + NET_EPOCH_ENTER(); IFP_TO_IA(ifp, ia, &in_ifa_tracker); - if (ia != NULL) { + if (ia != NULL) ip->ip_src = ia->ia_addr.sin_addr; - ifa_free(&ia->ia_ifa); - } + NET_EPOCH_EXIT(); } ip->ip_dst.s_addr = htonl(INADDR_ALLRPTS_GROUP); @@ -3634,7 +3616,6 @@ DB_SHOW_COMMAND(igi_list, db_show_igi_list) db_printf(" qi %u\n", igi->igi_qi); db_printf(" qri %u\n", igi->igi_qri); db_printf(" uri %u\n", igi->igi_uri); - /* SLIST_HEAD(,in_multi) igi_relinmhead */ /* struct mbufq igi_gq; */ db_printf("\n"); } diff --git a/freebsd/sys/netinet/igmp_var.h b/freebsd/sys/netinet/igmp_var.h index 4f9db06c..11f086f8 100644 --- a/freebsd/sys/netinet/igmp_var.h +++ b/freebsd/sys/netinet/igmp_var.h @@ -214,7 +214,6 @@ struct igmp_ifsoftc { uint32_t igi_qi; /* IGMPv3 Query Interval (s) */ uint32_t igi_qri; /* IGMPv3 Query Response Interval (s) */ uint32_t igi_uri; /* IGMPv3 Unsolicited Report Interval (s) */ - SLIST_HEAD(,in_multi) igi_relinmhead; /* released groups */ struct mbufq igi_gq; /* general query responses queue */ }; diff --git a/freebsd/sys/netinet/in.c b/freebsd/sys/netinet/in.c index 28c257aa..7233f9a2 100644 --- a/freebsd/sys/netinet/in.c +++ b/freebsd/sys/netinet/in.c @@ -104,7 +104,7 @@ in_localaddr(struct in_addr in) struct in_ifaddr *ia; IN_IFADDR_RLOCK(&in_ifa_tracker); - TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { + CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { if ((i & ia->ia_subnetmask) == ia->ia_subnet) { IN_IFADDR_RUNLOCK(&in_ifa_tracker); return (1); @@ -145,7 +145,7 @@ in_ifhasaddr(struct ifnet *ifp, struct in_addr in) struct in_ifaddr *ia; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; ia = (struct in_ifaddr *)ifa; @@ -282,7 +282,7 @@ in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, * first one on the interface, if possible. */ IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; ia = (struct in_ifaddr *)ifa; @@ -290,7 +290,7 @@ in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, break; } if (ifa == NULL) - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) if (ifa->ifa_addr->sa_family == AF_INET) { ia = (struct in_ifaddr *)ifa; if (prison_check_ip4(td->td_ucred, @@ -381,7 +381,7 @@ in_aifaddr_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp, struct thread *td) iaIsFirst = true; ia = NULL; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { struct in_ifaddr *it; if (ifa->ifa_addr->sa_family != AF_INET) @@ -459,12 +459,12 @@ in_aifaddr_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp, struct thread *td) /* if_addrhead is already referenced by ifa_alloc() */ IF_ADDR_WLOCK(ifp); - TAILQ_INSERT_TAIL(&ifp->if_addrhead, ifa, ifa_link); + CK_STAILQ_INSERT_TAIL(&ifp->if_addrhead, ifa, ifa_link); IF_ADDR_WUNLOCK(ifp); ifa_ref(ifa); /* in_ifaddrhead */ IN_IFADDR_WLOCK(); - TAILQ_INSERT_TAIL(&V_in_ifaddrhead, ia, ia_link); + CK_STAILQ_INSERT_TAIL(&V_in_ifaddrhead, ia, ia_link); LIST_INSERT_HEAD(INADDR_HASH(ia->ia_addr.sin_addr.s_addr), ia, ia_hash); IN_IFADDR_WUNLOCK(); @@ -537,12 +537,12 @@ fail1: (*carp_detach_p)(&ia->ia_ifa, false); IF_ADDR_WLOCK(ifp); - TAILQ_REMOVE(&ifp->if_addrhead, &ia->ia_ifa, ifa_link); + CK_STAILQ_REMOVE(&ifp->if_addrhead, &ia->ia_ifa, ifaddr, ifa_link); IF_ADDR_WUNLOCK(ifp); ifa_free(&ia->ia_ifa); /* if_addrhead */ IN_IFADDR_WLOCK(); - TAILQ_REMOVE(&V_in_ifaddrhead, ia, ia_link); + CK_STAILQ_REMOVE(&V_in_ifaddrhead, ia, in_ifaddr, ia_link); LIST_REMOVE(ia, ia_hash); IN_IFADDR_WUNLOCK(); ifa_free(&ia->ia_ifa); /* in_ifaddrhead */ @@ -576,7 +576,7 @@ in_difaddr_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp, struct thread *td) iaIsLast = true; ia = NULL; IF_ADDR_WLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { struct in_ifaddr *it; if (ifa->ifa_addr->sa_family != AF_INET) @@ -601,12 +601,12 @@ in_difaddr_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp, struct thread *td) return (EADDRNOTAVAIL); } - TAILQ_REMOVE(&ifp->if_addrhead, &ia->ia_ifa, ifa_link); + CK_STAILQ_REMOVE(&ifp->if_addrhead, &ia->ia_ifa, ifaddr, ifa_link); IF_ADDR_WUNLOCK(ifp); ifa_free(&ia->ia_ifa); /* if_addrhead */ IN_IFADDR_WLOCK(); - TAILQ_REMOVE(&V_in_ifaddrhead, ia, ia_link); + CK_STAILQ_REMOVE(&V_in_ifaddrhead, ia, in_ifaddr, ia_link); LIST_REMOVE(ia, ia_hash); IN_IFADDR_WUNLOCK(); @@ -636,12 +636,10 @@ in_difaddr_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp, struct thread *td) struct in_ifinfo *ii; ii = ((struct in_ifinfo *)ifp->if_afdata[AF_INET]); - IN_MULTI_LOCK(); if (ii->ii_allhosts) { - (void)in_leavegroup_locked(ii->ii_allhosts, NULL); + (void)in_leavegroup(ii->ii_allhosts, NULL); ii->ii_allhosts = NULL; } - IN_MULTI_UNLOCK(); } IF_ADDR_WLOCK(ifp); @@ -682,7 +680,7 @@ in_addprefix(struct in_ifaddr *target, int flags) IN_IFADDR_RLOCK(&in_ifa_tracker); /* Look for an existing address with the same prefix, mask, and fib */ - TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { + CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { if (rtinitflags(ia)) { p = ia->ia_dstaddr.sin_addr; @@ -842,7 +840,7 @@ in_scrubprefix(struct in_ifaddr *target, u_int flags) } IN_IFADDR_RLOCK(&in_ifa_tracker); - TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { + CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { if (rtinitflags(ia)) { p = ia->ia_dstaddr.sin_addr; @@ -918,10 +916,10 @@ in_ifscrub_all(void) struct ifaliasreq ifr; IFNET_RLOCK(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { /* Cannot lock here - lock recursion. */ /* IF_ADDR_RLOCK(ifp); */ - TAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, nifa) { + CK_STAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, nifa) { if (ifa->ifa_addr->sa_family != AF_INET) continue; @@ -982,7 +980,7 @@ in_broadcast(struct in_addr in, struct ifnet *ifp) * with a broadcast address. */ IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) if (ifa->ifa_addr->sa_family == AF_INET && in_ifaddr_broadcast(in, (struct in_ifaddr *)ifa)) { found = 1; @@ -998,11 +996,12 @@ in_broadcast(struct in_addr in, struct ifnet *ifp) void in_ifdetach(struct ifnet *ifp) { - + IN_MULTI_LOCK(); in_pcbpurgeif0(&V_ripcbinfo, ifp); in_pcbpurgeif0(&V_udbinfo, ifp); in_pcbpurgeif0(&V_ulitecbinfo, ifp); in_purgemaddrs(ifp); + IN_MULTI_UNLOCK(); } /* @@ -1015,12 +1014,12 @@ in_ifdetach(struct ifnet *ifp) static void in_purgemaddrs(struct ifnet *ifp) { - LIST_HEAD(,in_multi) purgeinms; - struct in_multi *inm, *tinm; - struct ifmultiaddr *ifma; + struct in_multi_head purgeinms; + struct in_multi *inm; + struct ifmultiaddr *ifma, *next; - LIST_INIT(&purgeinms); - IN_MULTI_LOCK(); + SLIST_INIT(&purgeinms); + IN_MULTI_LIST_LOCK(); /* * Extract list of in_multi associated with the detaching ifp @@ -1028,27 +1027,24 @@ in_purgemaddrs(struct ifnet *ifp) * We need to do this as IF_ADDR_LOCK() may be re-acquired * by code further down. */ - IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + IF_ADDR_WLOCK(ifp); + restart: + CK_STAILQ_FOREACH_SAFE(ifma, &ifp->if_multiaddrs, ifma_link, next) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) continue; -#if 0 - KASSERT(ifma->ifma_protospec != NULL, - ("%s: ifma_protospec is NULL", __func__)); -#endif inm = (struct in_multi *)ifma->ifma_protospec; - LIST_INSERT_HEAD(&purgeinms, inm, inm_link); + inm_rele_locked(&purgeinms, inm); + if (__predict_false(ifma_restart)) { + ifma_restart = true; + goto restart; + } } - IF_ADDR_RUNLOCK(ifp); + IF_ADDR_WUNLOCK(ifp); - LIST_FOREACH_SAFE(inm, &purgeinms, inm_link, tinm) { - LIST_REMOVE(inm, inm_link); - inm_release_locked(inm); - } + inm_release_list_deferred(&purgeinms); igmp_ifdetach(ifp); - - IN_MULTI_UNLOCK(); + IN_MULTI_LIST_UNLOCK(); } struct in_llentry { @@ -1063,9 +1059,11 @@ struct in_llentry { * Do actual deallocation of @lle. */ static void -in_lltable_destroy_lle_unlocked(struct llentry *lle) +in_lltable_destroy_lle_unlocked(epoch_context_t ctx) { + struct llentry *lle; + lle = __containerof(ctx, struct llentry, lle_epoch_ctx); LLE_LOCK_DESTROY(lle); LLE_REQ_DESTROY(lle); free(lle, M_LLTABLE); @@ -1093,7 +1091,7 @@ in_lltable_destroy_lle(struct llentry *lle) { LLE_WUNLOCK(lle); - in_lltable_destroy_lle_unlocked(lle); + epoch_call(net_epoch_preempt, &lle->lle_epoch_ctx, in_lltable_destroy_lle_unlocked); } static struct llentry * @@ -1160,7 +1158,6 @@ in_lltable_match_prefix(const struct sockaddr *saddr, static void in_lltable_free_entry(struct lltable *llt, struct llentry *lle) { - struct ifnet *ifp; size_t pkts_dropped; LLE_WLOCK_ASSERT(lle); @@ -1168,8 +1165,7 @@ in_lltable_free_entry(struct lltable *llt, struct llentry *lle) /* Unlink entry from table if not already */ if ((lle->la_flags & LLE_LINKED) != 0) { - ifp = llt->llt_ifp; - IF_AFDATA_WLOCK_ASSERT(ifp); + IF_AFDATA_WLOCK_ASSERT(llt->llt_ifp); lltable_unlink_entry(llt, lle); } @@ -1304,7 +1300,7 @@ in_lltable_find_dst(struct lltable *llt, struct in_addr dst) hashidx = in_lltable_hash_dst(dst, llt->llt_hsize); lleh = &llt->lle_head[hashidx]; - LIST_FOREACH(lle, lleh, lle_next) { + CK_LIST_FOREACH(lle, lleh, lle_next) { if (lle->la_flags & LLE_DELETED) continue; if (lle->r_l3addr.addr4.s_addr == dst.s_addr) @@ -1360,7 +1356,7 @@ in_lltable_alloc(struct lltable *llt, u_int flags, const struct sockaddr *l3addr linkhdrsize = LLE_MAX_LINKHDR; if (lltable_calc_llheader(ifp, AF_INET, IF_LLADDR(ifp), linkhdr, &linkhdrsize, &lladdr_off) != 0) { - in_lltable_destroy_lle_unlocked(lle); + epoch_call(net_epoch_preempt, &lle->lle_epoch_ctx, in_lltable_destroy_lle_unlocked); return (NULL); } lltable_set_entry_addr(ifp, lle, linkhdr, linkhdrsize, @@ -1420,52 +1416,51 @@ in_lltable_dump_entry(struct lltable *llt, struct llentry *lle, int error; bzero(&arpc, sizeof(arpc)); - /* skip deleted entries */ - if ((lle->la_flags & LLE_DELETED) == LLE_DELETED) - return (0); - /* Skip if jailed and not a valid IP of the prison. */ - lltable_fill_sa_entry(lle,(struct sockaddr *)&arpc.sin); - if (prison_if(wr->td->td_ucred, - (struct sockaddr *)&arpc.sin) != 0) - return (0); - /* - * produce a msg made of: - * struct rt_msghdr; - * struct sockaddr_in; (IPv4) - * struct sockaddr_dl; - */ - arpc.rtm.rtm_msglen = sizeof(arpc); - arpc.rtm.rtm_version = RTM_VERSION; - arpc.rtm.rtm_type = RTM_GET; - arpc.rtm.rtm_flags = RTF_UP; - arpc.rtm.rtm_addrs = RTA_DST | RTA_GATEWAY; - - /* publish */ - if (lle->la_flags & LLE_PUB) - arpc.rtm.rtm_flags |= RTF_ANNOUNCE; - - sdl = &arpc.sdl; - sdl->sdl_family = AF_LINK; - sdl->sdl_len = sizeof(*sdl); - sdl->sdl_index = ifp->if_index; - sdl->sdl_type = ifp->if_type; - if ((lle->la_flags & LLE_VALID) == LLE_VALID) { - sdl->sdl_alen = ifp->if_addrlen; - bcopy(lle->ll_addr, LLADDR(sdl), ifp->if_addrlen); - } else { - sdl->sdl_alen = 0; - bzero(LLADDR(sdl), ifp->if_addrlen); - } + /* skip deleted entries */ + if ((lle->la_flags & LLE_DELETED) == LLE_DELETED) + return (0); + /* Skip if jailed and not a valid IP of the prison. */ + lltable_fill_sa_entry(lle,(struct sockaddr *)&arpc.sin); + if (prison_if(wr->td->td_ucred, (struct sockaddr *)&arpc.sin) != 0) + return (0); + /* + * produce a msg made of: + * struct rt_msghdr; + * struct sockaddr_in; (IPv4) + * struct sockaddr_dl; + */ + arpc.rtm.rtm_msglen = sizeof(arpc); + arpc.rtm.rtm_version = RTM_VERSION; + arpc.rtm.rtm_type = RTM_GET; + arpc.rtm.rtm_flags = RTF_UP; + arpc.rtm.rtm_addrs = RTA_DST | RTA_GATEWAY; + + /* publish */ + if (lle->la_flags & LLE_PUB) + arpc.rtm.rtm_flags |= RTF_ANNOUNCE; + + sdl = &arpc.sdl; + sdl->sdl_family = AF_LINK; + sdl->sdl_len = sizeof(*sdl); + sdl->sdl_index = ifp->if_index; + sdl->sdl_type = ifp->if_type; + if ((lle->la_flags & LLE_VALID) == LLE_VALID) { + sdl->sdl_alen = ifp->if_addrlen; + bcopy(lle->ll_addr, LLADDR(sdl), ifp->if_addrlen); + } else { + sdl->sdl_alen = 0; + bzero(LLADDR(sdl), ifp->if_addrlen); + } - arpc.rtm.rtm_rmx.rmx_expire = - lle->la_flags & LLE_STATIC ? 0 : lle->la_expire; - arpc.rtm.rtm_flags |= (RTF_HOST | RTF_LLDATA); - if (lle->la_flags & LLE_STATIC) - arpc.rtm.rtm_flags |= RTF_STATIC; - if (lle->la_flags & LLE_IFADDR) - arpc.rtm.rtm_flags |= RTF_PINNED; - arpc.rtm.rtm_index = ifp->if_index; - error = SYSCTL_OUT(wr, &arpc, sizeof(arpc)); + arpc.rtm.rtm_rmx.rmx_expire = + lle->la_flags & LLE_STATIC ? 0 : lle->la_expire; + arpc.rtm.rtm_flags |= (RTF_HOST | RTF_LLDATA); + if (lle->la_flags & LLE_STATIC) + arpc.rtm.rtm_flags |= RTF_STATIC; + if (lle->la_flags & LLE_IFADDR) + arpc.rtm.rtm_flags |= RTF_PINNED; + arpc.rtm.rtm_index = ifp->if_index; + error = SYSCTL_OUT(wr, &arpc, sizeof(arpc)); return (error); } diff --git a/freebsd/sys/netinet/in_mcast.c b/freebsd/sys/netinet/in_mcast.c index 41beed9b..ea4779fc 100644 --- a/freebsd/sys/netinet/in_mcast.c +++ b/freebsd/sys/netinet/in_mcast.c @@ -53,6 +53,7 @@ __FBSDID("$FreeBSD$"); #include <sys/sysctl.h> #include <sys/ktr.h> #include <sys/taskqueue.h> +#include <sys/gtaskqueue.h> #include <sys/tree.h> #include <net/if.h> @@ -61,6 +62,8 @@ __FBSDID("$FreeBSD$"); #include <net/route.h> #include <net/vnet.h> +#include <net/ethernet.h> + #include <netinet/in.h> #include <netinet/in_systm.h> #include <netinet/in_fib.h> @@ -93,17 +96,25 @@ static MALLOC_DEFINE(M_IPMSOURCE, "ip_msource", /* * Locking: - * - Lock order is: Giant, INP_WLOCK, IN_MULTI_LOCK, IGMP_LOCK, IF_ADDR_LOCK. + * - Lock order is: Giant, INP_WLOCK, IN_MULTI_LIST_LOCK, IGMP_LOCK, IF_ADDR_LOCK. * - The IF_ADDR_LOCK is implicitly taken by inm_lookup() earlier, however * it can be taken by code in net/if.c also. * - ip_moptions and in_mfilter are covered by the INP_WLOCK. * - * struct in_multi is covered by IN_MULTI_LOCK. There isn't strictly + * struct in_multi is covered by IN_MULTI_LIST_LOCK. There isn't strictly * any need for in_multi itself to be virtualized -- it is bound to an ifp * anyway no matter what happens. */ -struct mtx in_multi_mtx; -MTX_SYSINIT(in_multi_mtx, &in_multi_mtx, "in_multi_mtx", MTX_DEF); +struct mtx in_multi_list_mtx; +MTX_SYSINIT(in_multi_mtx, &in_multi_list_mtx, "in_multi_list_mtx", MTX_DEF); + +struct mtx in_multi_free_mtx; +MTX_SYSINIT(in_multi_free_mtx, &in_multi_free_mtx, "in_multi_free_mtx", MTX_DEF); + +struct sx in_multi_sx; +SX_SYSINIT(in_multi_sx, &in_multi_sx, "in_multi_sx"); + +int ifma_restart; /* * Functions with non-static linkage defined in this file should be @@ -153,10 +164,9 @@ static int inm_is_ifp_detached(const struct in_multi *); static int inm_merge(struct in_multi *, /*const*/ struct in_mfilter *); static void inm_purge(struct in_multi *); static void inm_reap(struct in_multi *); +static void inm_release(struct in_multi *); static struct ip_moptions * inp_findmoptions(struct inpcb *); -static void inp_freemoptions_internal(struct ip_moptions *); -static void inp_gcmoptions(void *, int); static int inp_get_source_filters(struct inpcb *, struct sockopt *); static int inp_join_group(struct inpcb *, struct sockopt *); static int inp_leave_group(struct inpcb *, struct sockopt *); @@ -189,10 +199,6 @@ static SYSCTL_NODE(_net_inet_ip_mcast, OID_AUTO, filters, CTLFLAG_RD | CTLFLAG_MPSAFE, sysctl_ip_mcast_filters, "Per-interface stack-wide source filters"); -static STAILQ_HEAD(, ip_moptions) imo_gc_list = - STAILQ_HEAD_INITIALIZER(imo_gc_list); -static struct task imo_gc_task = TASK_INITIALIZER(0, inp_gcmoptions, NULL); - #ifdef KTR /* * Inline function which wraps assertions for a valid ifp. @@ -218,6 +224,93 @@ inm_is_ifp_detached(const struct in_multi *inm) } #endif +static struct grouptask free_gtask; +static struct in_multi_head inm_free_list; +static void inm_release_task(void *arg __unused); +static void inm_init(void) +{ + SLIST_INIT(&inm_free_list); + taskqgroup_config_gtask_init(NULL, &free_gtask, inm_release_task, "inm release task"); +} + +SYSINIT(inm_init, SI_SUB_SMP + 1, SI_ORDER_FIRST, + inm_init, NULL); + + +void +inm_release_list_deferred(struct in_multi_head *inmh) +{ + + if (SLIST_EMPTY(inmh)) + return; + mtx_lock(&in_multi_free_mtx); + SLIST_CONCAT(&inm_free_list, inmh, in_multi, inm_nrele); + mtx_unlock(&in_multi_free_mtx); + GROUPTASK_ENQUEUE(&free_gtask); +} + +void +inm_disconnect(struct in_multi *inm) +{ + struct ifnet *ifp; + struct ifmultiaddr *ifma, *ll_ifma; + + ifp = inm->inm_ifp; + IF_ADDR_WLOCK_ASSERT(ifp); + ifma = inm->inm_ifma; + + if_ref(ifp); + CK_STAILQ_REMOVE(&ifp->if_multiaddrs, ifma, ifmultiaddr, ifma_link); + MCDPRINTF("removed ifma: %p from %s\n", ifma, ifp->if_xname); + if ((ll_ifma = ifma->ifma_llifma) != NULL) { + MPASS(ifma != ll_ifma); + ifma->ifma_llifma = NULL; + MPASS(ll_ifma->ifma_llifma == NULL); + MPASS(ll_ifma->ifma_ifp == ifp); + if (--ll_ifma->ifma_refcount == 0) { + CK_STAILQ_REMOVE(&ifp->if_multiaddrs, ll_ifma, ifmultiaddr, ifma_link); + MCDPRINTF("removed ll_ifma: %p from %s\n", ll_ifma, ifp->if_xname); + if_freemulti(ll_ifma); + ifma_restart = true; + } + } +} + +void +inm_release_deferred(struct in_multi *inm) +{ + struct in_multi_head tmp; + + IN_MULTI_LIST_LOCK_ASSERT(); + MPASS(inm->inm_refcount > 0); + if (--inm->inm_refcount == 0) { + SLIST_INIT(&tmp); + inm_disconnect(inm); + inm->inm_ifma->ifma_protospec = NULL; + SLIST_INSERT_HEAD(&tmp, inm, inm_nrele); + inm_release_list_deferred(&tmp); + } +} + +static void +inm_release_task(void *arg __unused) +{ + struct in_multi_head inm_free_tmp; + struct in_multi *inm, *tinm; + + SLIST_INIT(&inm_free_tmp); + mtx_lock(&in_multi_free_mtx); + SLIST_CONCAT(&inm_free_tmp, &inm_free_list, in_multi, inm_nrele); + mtx_unlock(&in_multi_free_mtx); + IN_MULTI_LOCK(); + SLIST_FOREACH_SAFE(inm, &inm_free_tmp, inm_nrele, tinm) { + SLIST_REMOVE_HEAD(&inm_free_tmp, inm_nrele); + MPASS(inm); + inm_release(inm); + } + IN_MULTI_UNLOCK(); +} + /* * Initialize an in_mfilter structure to a known state at t0, t1 * with an empty source filter list. @@ -234,7 +327,7 @@ imf_init(struct in_mfilter *imf, const int st0, const int st1) /* * Function for looking up an in_multi record for an IPv4 multicast address * on a given interface. ifp must be valid. If no record found, return NULL. - * The IN_MULTI_LOCK and IF_ADDR_LOCK on ifp must be held. + * The IN_MULTI_LIST_LOCK and IF_ADDR_LOCK on ifp must be held. */ struct in_multi * inm_lookup_locked(struct ifnet *ifp, const struct in_addr ina) @@ -242,17 +335,18 @@ inm_lookup_locked(struct ifnet *ifp, const struct in_addr ina) struct ifmultiaddr *ifma; struct in_multi *inm; - IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_LOCK_ASSERT(); IF_ADDR_LOCK_ASSERT(ifp); inm = NULL; - TAILQ_FOREACH(ifma, &((ifp)->if_multiaddrs), ifma_link) { - if (ifma->ifma_addr->sa_family == AF_INET) { - inm = (struct in_multi *)ifma->ifma_protospec; - if (inm->inm_addr.s_addr == ina.s_addr) - break; - inm = NULL; - } + CK_STAILQ_FOREACH(ifma, &((ifp)->if_multiaddrs), ifma_link) { + if (ifma->ifma_addr->sa_family != AF_INET || + ifma->ifma_protospec == NULL) + continue; + inm = (struct in_multi *)ifma->ifma_protospec; + if (inm->inm_addr.s_addr == ina.s_addr) + break; + inm = NULL; } return (inm); } @@ -266,7 +360,7 @@ inm_lookup(struct ifnet *ifp, const struct in_addr ina) { struct in_multi *inm; - IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_LOCK_ASSERT(); IF_ADDR_RLOCK(ifp); inm = inm_lookup_locked(ifp, ina); IF_ADDR_RUNLOCK(ifp); @@ -453,7 +547,7 @@ in_getmulti(struct ifnet *ifp, const struct in_addr *group, IN_MULTI_LOCK_ASSERT(); ii = (struct in_ifinfo *)ifp->if_afdata[AF_INET]; - + IN_MULTI_LIST_LOCK(); inm = inm_lookup(ifp, *group); if (inm != NULL) { /* @@ -462,11 +556,13 @@ in_getmulti(struct ifnet *ifp, const struct in_addr *group, */ KASSERT(inm->inm_refcount >= 1, ("%s: bad refcount %d", __func__, inm->inm_refcount)); - ++inm->inm_refcount; + inm_acquire_locked(inm); *pinm = inm; - return (0); } - + IN_MULTI_LIST_UNLOCK(); + if (inm != NULL) + return (0); + memset(&gsin, 0, sizeof(gsin)); gsin.sin_family = AF_INET; gsin.sin_len = sizeof(struct sockaddr_in); @@ -481,6 +577,7 @@ in_getmulti(struct ifnet *ifp, const struct in_addr *group, return (error); /* XXX ifma_protospec must be covered by IF_ADDR_LOCK */ + IN_MULTI_LIST_LOCK(); IF_ADDR_WLOCK(ifp); /* @@ -506,10 +603,9 @@ in_getmulti(struct ifnet *ifp, const struct in_addr *group, __func__, ifma, inm, inet_ntoa_r(*group, addrbuf)); } #endif - ++inm->inm_refcount; + inm_acquire_locked(inm); *pinm = inm; - IF_ADDR_WUNLOCK(ifp); - return (0); + goto out_locked; } IF_ADDR_WLOCK_ASSERT(ifp); @@ -524,6 +620,7 @@ in_getmulti(struct ifnet *ifp, const struct in_addr *group, inm = malloc(sizeof(*inm), M_IPMADDR, M_NOWAIT | M_ZERO); if (inm == NULL) { IF_ADDR_WUNLOCK(ifp); + IN_MULTI_LIST_UNLOCK(); if_delmulti_ifma(ifma); return (ENOMEM); } @@ -541,8 +638,9 @@ in_getmulti(struct ifnet *ifp, const struct in_addr *group, ifma->ifma_protospec = inm; *pinm = inm; - + out_locked: IF_ADDR_WUNLOCK(ifp); + IN_MULTI_LIST_UNLOCK(); return (0); } @@ -552,36 +650,33 @@ in_getmulti(struct ifnet *ifp, const struct in_addr *group, * If the refcount drops to 0, free the in_multi record and * delete the underlying link-layer membership. */ -void -inm_release_locked(struct in_multi *inm) +static void +inm_release(struct in_multi *inm) { struct ifmultiaddr *ifma; - - IN_MULTI_LOCK_ASSERT(); + struct ifnet *ifp; CTR2(KTR_IGMPV3, "%s: refcount is %d", __func__, inm->inm_refcount); - - if (--inm->inm_refcount > 0) { - CTR2(KTR_IGMPV3, "%s: refcount is now %d", __func__, - inm->inm_refcount); - return; - } - + MPASS(inm->inm_refcount == 0); CTR2(KTR_IGMPV3, "%s: freeing inm %p", __func__, inm); ifma = inm->inm_ifma; + ifp = inm->inm_ifp; /* XXX this access is not covered by IF_ADDR_LOCK */ CTR2(KTR_IGMPV3, "%s: purging ifma %p", __func__, ifma); - KASSERT(ifma->ifma_protospec == inm, - ("%s: ifma_protospec != inm", __func__)); - ifma->ifma_protospec = NULL; - - inm_purge(inm); - - free(inm, M_IPMADDR); - - if_delmulti_ifma(ifma); + if (ifp != NULL) { + CURVNET_SET(ifp->if_vnet); + inm_purge(inm); + free(inm, M_IPMADDR); + if_delmulti_ifma_flags(ifma, 1); + CURVNET_RESTORE(); + if_rele(ifp); + } else { + inm_purge(inm); + free(inm, M_IPMADDR); + if_delmulti_ifma_flags(ifma, 1); + } } /* @@ -594,7 +689,7 @@ inm_clear_recorded(struct in_multi *inm) { struct ip_msource *ims; - IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_LOCK_ASSERT(); RB_FOREACH(ims, ip_msource_tree, &inm->inm_srcs) { if (ims->ims_stp) { @@ -634,7 +729,7 @@ inm_record_source(struct in_multi *inm, const in_addr_t naddr) struct ip_msource find; struct ip_msource *ims, *nims; - IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_LOCK_ASSERT(); find.ims_haddr = ntohl(naddr); ims = RB_FIND(ip_msource_tree, &inm->inm_srcs, &find); @@ -961,6 +1056,7 @@ inm_merge(struct in_multi *inm, /*const*/ struct in_mfilter *imf) schanged = 0; error = 0; nsrc1 = nsrc0 = 0; + IN_MULTI_LIST_LOCK_ASSERT(); /* * Update the source filters first, as this may fail. @@ -1167,6 +1263,7 @@ in_joingroup_locked(struct ifnet *ifp, const struct in_addr *gina, int error; IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_UNLOCK_ASSERT(); CTR4(KTR_IGMPV3, "%s: join 0x%08x on %p(%s))", __func__, ntohl(gina->s_addr), ifp, ifp->if_xname); @@ -1188,7 +1285,7 @@ in_joingroup_locked(struct ifnet *ifp, const struct in_addr *gina, CTR1(KTR_IGMPV3, "%s: in_getmulti() failure", __func__); return (error); } - + IN_MULTI_LIST_LOCK(); CTR1(KTR_IGMPV3, "%s: merge inm state", __func__); error = inm_merge(inm, imf); if (error) { @@ -1203,13 +1300,15 @@ in_joingroup_locked(struct ifnet *ifp, const struct in_addr *gina, goto out_inm_release; } -out_inm_release: + out_inm_release: if (error) { + CTR2(KTR_IGMPV3, "%s: dropping ref on %p", __func__, inm); - inm_release_locked(inm); + inm_release_deferred(inm); } else { *pinm = inm; } + IN_MULTI_LIST_UNLOCK(); return (error); } @@ -1251,6 +1350,7 @@ in_leavegroup_locked(struct in_multi *inm, /*const*/ struct in_mfilter *imf) error = 0; IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_UNLOCK_ASSERT(); CTR5(KTR_IGMPV3, "%s: leave inm %p, 0x%08x/%s, imf %p", __func__, inm, ntohl(inm->inm_addr.s_addr), @@ -1274,18 +1374,22 @@ in_leavegroup_locked(struct in_multi *inm, /*const*/ struct in_mfilter *imf) * the transaction, it MUST NOT fail. */ CTR1(KTR_IGMPV3, "%s: merge inm state", __func__); + IN_MULTI_LIST_LOCK(); error = inm_merge(inm, imf); KASSERT(error == 0, ("%s: failed to merge inm state", __func__)); CTR1(KTR_IGMPV3, "%s: doing igmp downcall", __func__); CURVNET_SET(inm->inm_ifp->if_vnet); error = igmp_change_state(inm); + IF_ADDR_WLOCK(inm->inm_ifp); + inm_release_deferred(inm); + IF_ADDR_WUNLOCK(inm->inm_ifp); + IN_MULTI_LIST_UNLOCK(); CURVNET_RESTORE(); if (error) CTR1(KTR_IGMPV3, "%s: failed igmp downcall", __func__); CTR2(KTR_IGMPV3, "%s: dropping ref on %p", __func__, inm); - inm_release_locked(inm); return (error); } @@ -1317,18 +1421,6 @@ in_addmulti(struct in_addr *ap, struct ifnet *ifp) } /* - * Leave an IPv4 multicast group, assumed to be in exclusive (*,G) mode. - * This KPI is for legacy kernel consumers only. - */ -void -in_delmulti(struct in_multi *inm) -{ - - (void)in_leavegroup(inm, NULL); -} -/*#endif*/ - -/* * Block or unblock an ASM multicast source on an inpcb. * This implements the delta-based API described in RFC 3678. * @@ -1489,7 +1581,7 @@ inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) * Begin state merge transaction at IGMP layer. */ IN_MULTI_LOCK(); - + IN_MULTI_LIST_LOCK(); CTR1(KTR_IGMPV3, "%s: merge inm state", __func__); error = inm_merge(inm, imf); if (error) { @@ -1505,7 +1597,7 @@ inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) out_in_multi_locked: IN_MULTI_UNLOCK(); - + IN_MULTI_UNLOCK(); out_imf_rollback: if (error) imf_rollback(imf); @@ -1571,37 +1663,31 @@ inp_findmoptions(struct inpcb *inp) return (imo); } -/* - * Discard the IP multicast options (and source filters). To minimize - * the amount of work done while holding locks such as the INP's - * pcbinfo lock (which is used in the receive path), the free - * operation is performed asynchronously in a separate task. - * - * SMPng: NOTE: assumes INP write lock is held. - */ -void -inp_freemoptions(struct ip_moptions *imo) -{ - - KASSERT(imo != NULL, ("%s: ip_moptions is NULL", __func__)); - IN_MULTI_LOCK(); - STAILQ_INSERT_TAIL(&imo_gc_list, imo, imo_link); - IN_MULTI_UNLOCK(); - taskqueue_enqueue(taskqueue_thread, &imo_gc_task); -} - static void -inp_freemoptions_internal(struct ip_moptions *imo) +inp_gcmoptions(epoch_context_t ctx) { + struct ip_moptions *imo; struct in_mfilter *imf; + struct in_multi *inm; + struct ifnet *ifp; size_t idx, nmships; + imo = __containerof(ctx, struct ip_moptions, imo_epoch_ctx); + nmships = imo->imo_num_memberships; for (idx = 0; idx < nmships; ++idx) { imf = imo->imo_mfilters ? &imo->imo_mfilters[idx] : NULL; if (imf) imf_leave(imf); - (void)in_leavegroup(imo->imo_membership[idx], imf); + inm = imo->imo_membership[idx]; + ifp = inm->inm_ifp; + if (ifp != NULL) { + CURVNET_SET(ifp->if_vnet); + (void)in_leavegroup(inm, imf); + CURVNET_RESTORE(); + } else { + (void)in_leavegroup(inm, imf); + } if (imf) imf_purge(imf); } @@ -1612,20 +1698,18 @@ inp_freemoptions_internal(struct ip_moptions *imo) free(imo, M_IPMOPTS); } -static void -inp_gcmoptions(void *context, int pending) +/* + * Discard the IP multicast options (and source filters). To minimize + * the amount of work done while holding locks such as the INP's + * pcbinfo lock (which is used in the receive path), the free + * operation is deferred to the epoch callback task. + */ +void +inp_freemoptions(struct ip_moptions *imo) { - struct ip_moptions *imo; - - IN_MULTI_LOCK(); - while (!STAILQ_EMPTY(&imo_gc_list)) { - imo = STAILQ_FIRST(&imo_gc_list); - STAILQ_REMOVE_HEAD(&imo_gc_list, imo_link); - IN_MULTI_UNLOCK(); - inp_freemoptions_internal(imo); - IN_MULTI_LOCK(); - } - IN_MULTI_UNLOCK(); + if (imo == NULL) + return; + epoch_call(net_epoch_preempt, &imo->imo_epoch_ctx, inp_gcmoptions); } /* @@ -1794,12 +1878,12 @@ inp_getmoptions(struct inpcb *inp, struct sockopt *sopt) mreqn.imr_address = imo->imo_multicast_addr; } else if (ifp != NULL) { mreqn.imr_ifindex = ifp->if_index; + NET_EPOCH_ENTER(); IFP_TO_IA(ifp, ia, &in_ifa_tracker); - if (ia != NULL) { + if (ia != NULL) mreqn.imr_address = IA_SIN(ia)->sin_addr; - ifa_free(&ia->ia_ifa); - } + NET_EPOCH_EXIT(); } } INP_WUNLOCK(inp); @@ -1907,7 +1991,7 @@ inp_lookup_mcast_ifp(const struct inpcb *inp, mifp = NULL; IN_IFADDR_RLOCK(&in_ifa_tracker); - TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { + CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { mifp = ia->ia_ifp; if (!(mifp->if_flags & IFF_LOOPBACK) && (mifp->if_flags & IFF_MULTICAST)) { @@ -2165,6 +2249,8 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) /* * Begin state merge transaction at IGMP layer. */ + in_pcbref(inp); + INP_WUNLOCK(inp); IN_MULTI_LOCK(); if (is_new) { @@ -2173,20 +2259,23 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) if (error) { CTR1(KTR_IGMPV3, "%s: in_joingroup_locked failed", __func__); - IN_MULTI_UNLOCK(); + IN_MULTI_LIST_UNLOCK(); goto out_imo_free; } imo->imo_membership[idx] = inm; } else { CTR1(KTR_IGMPV3, "%s: merge inm state", __func__); + IN_MULTI_LIST_LOCK(); error = inm_merge(inm, imf); if (error) { CTR1(KTR_IGMPV3, "%s: failed to merge inm state", - __func__); + __func__); + IN_MULTI_LIST_UNLOCK(); goto out_in_multi_locked; } CTR1(KTR_IGMPV3, "%s: doing igmp downcall", __func__); error = igmp_change_state(inm); + IN_MULTI_LIST_UNLOCK(); if (error) { CTR1(KTR_IGMPV3, "%s: failed igmp downcall", __func__); @@ -2197,8 +2286,9 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) out_in_multi_locked: IN_MULTI_UNLOCK(); - - INP_WLOCK_ASSERT(inp); + INP_WLOCK(inp); + if (in_pcbrele_wlocked(inp)) + return (ENXIO); if (error) { imf_rollback(imf); if (is_new) @@ -2387,6 +2477,8 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) /* * Begin state merge transaction at IGMP layer. */ + in_pcbref(inp); + INP_WUNLOCK(inp); IN_MULTI_LOCK(); if (is_final) { @@ -2397,6 +2489,7 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) (void)in_leavegroup_locked(inm, imf); } else { CTR1(KTR_IGMPV3, "%s: merge inm state", __func__); + IN_MULTI_LIST_LOCK(); error = inm_merge(inm, imf); if (error) { CTR1(KTR_IGMPV3, "%s: failed to merge inm state", @@ -2406,6 +2499,7 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) CTR1(KTR_IGMPV3, "%s: doing igmp downcall", __func__); error = igmp_change_state(inm); + IN_MULTI_LIST_UNLOCK(); if (error) { CTR1(KTR_IGMPV3, "%s: failed igmp downcall", __func__); @@ -2415,6 +2509,9 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) out_in_multi_locked: IN_MULTI_UNLOCK(); + INP_WLOCK(inp); + if (in_pcbrele_wlocked(inp)) + return (ENXIO); if (error) imf_rollback(imf); @@ -2641,6 +2738,7 @@ inp_set_source_filters(struct inpcb *inp, struct sockopt *sopt) INP_WLOCK_ASSERT(inp); IN_MULTI_LOCK(); + IN_MULTI_LIST_LOCK(); /* * Begin state merge transaction at IGMP layer. @@ -2649,11 +2747,13 @@ inp_set_source_filters(struct inpcb *inp, struct sockopt *sopt) error = inm_merge(inm, imf); if (error) { CTR1(KTR_IGMPV3, "%s: failed to merge inm state", __func__); + IN_MULTI_LIST_UNLOCK(); goto out_in_multi_locked; } CTR1(KTR_IGMPV3, "%s: doing igmp downcall", __func__); error = igmp_change_state(inm); + IN_MULTI_LIST_UNLOCK(); if (error) CTR1(KTR_IGMPV3, "%s: failed igmp downcall", __func__); @@ -2885,10 +2985,10 @@ sysctl_ip_mcast_filters(SYSCTL_HANDLER_ARGS) if (retval) return (retval); - IN_MULTI_LOCK(); + IN_MULTI_LIST_LOCK(); IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) continue; @@ -2918,7 +3018,7 @@ sysctl_ip_mcast_filters(SYSCTL_HANDLER_ARGS) } IF_ADDR_RUNLOCK(ifp); - IN_MULTI_UNLOCK(); + IN_MULTI_LIST_UNLOCK(); return (retval); } diff --git a/freebsd/sys/netinet/in_pcb.c b/freebsd/sys/netinet/in_pcb.c index 0d388132..f89487b6 100644 --- a/freebsd/sys/netinet/in_pcb.c +++ b/freebsd/sys/netinet/in_pcb.c @@ -64,6 +64,7 @@ __FBSDID("$FreeBSD$"); #include <sys/domain.h> #include <sys/protosw.h> #include <sys/rmlock.h> +#include <sys/smp.h> #include <sys/socket.h> #include <sys/socketvar.h> #include <sys/sockio.h> @@ -93,6 +94,9 @@ __FBSDID("$FreeBSD$"); #include <netinet/in_pcb.h> #include <netinet/ip_var.h> #include <netinet/tcp_var.h> +#ifdef TCPHPTS +#include <netinet/tcp_hpts.h> +#endif #include <netinet/udp.h> #include <netinet/udp_var.h> #endif @@ -590,7 +594,7 @@ in_pcbbind_setup(struct inpcb *inp, struct sockaddr *nam, in_addr_t *laddrp, INP_LOCK_ASSERT(inp); INP_HASH_LOCK_ASSERT(pcbinfo); - if (TAILQ_EMPTY(&V_in_ifaddrhead)) /* XXX broken! */ + if (CK_STAILQ_EMPTY(&V_in_ifaddrhead)) /* XXX broken! */ return (EADDRNOTAVAIL); laddr.s_addr = *laddrp; if (nam != NULL && laddr.s_addr != INADDR_ANY) @@ -802,7 +806,6 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, int error; KASSERT(laddr != NULL, ("%s: laddr NULL", __func__)); - /* * Bypass source address selection and use the primary jail IP * if requested. @@ -835,15 +838,18 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, * network and try to find a corresponding interface to take * the source address from. */ + NET_EPOCH_ENTER(); if (sro.ro_rt == NULL || sro.ro_rt->rt_ifp == NULL) { struct in_ifaddr *ia; struct ifnet *ifp; ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin, inp->inp_socket->so_fibnum)); - if (ia == NULL) + if (ia == NULL) { ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0, inp->inp_socket->so_fibnum)); + + } if (ia == NULL) { error = ENETUNREACH; goto done; @@ -851,15 +857,13 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, if (cred == NULL || !prison_flag(cred, PR_IP4)) { laddr->s_addr = ia->ia_addr.sin_addr.s_addr; - ifa_free(&ia->ia_ifa); goto done; } ifp = ia->ia_ifp; - ifa_free(&ia->ia_ifa); ia = NULL; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { sa = ifa->ifa_addr; if (sa->sa_family != AF_INET) @@ -918,7 +922,7 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, ia = NULL; ifp = sro.ro_rt->rt_ifp; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { sa = ifa->ifa_addr; if (sa->sa_family != AF_INET) continue; @@ -972,7 +976,6 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, goto done; } laddr->s_addr = ia->ia_addr.sin_addr.s_addr; - ifa_free(&ia->ia_ifa); goto done; } @@ -981,10 +984,9 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, struct ifnet *ifp; ifp = ia->ia_ifp; - ifa_free(&ia->ia_ifa); ia = NULL; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { sa = ifa->ifa_addr; if (sa->sa_family != AF_INET) @@ -1010,6 +1012,7 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, } done: + NET_EPOCH_EXIT(); if (sro.ro_rt != NULL) RTFREE(sro.ro_rt); return (error); @@ -1063,7 +1066,7 @@ in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam, faddr = sin->sin_addr; fport = sin->sin_port; - if (!TAILQ_EMPTY(&V_in_ifaddrhead)) { + if (!CK_STAILQ_EMPTY(&V_in_ifaddrhead)) { /* * If the destination address is INADDR_ANY, * use the primary local address. @@ -1074,16 +1077,16 @@ in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam, if (faddr.s_addr == INADDR_ANY) { IN_IFADDR_RLOCK(&in_ifa_tracker); faddr = - IA_SIN(TAILQ_FIRST(&V_in_ifaddrhead))->sin_addr; + IA_SIN(CK_STAILQ_FIRST(&V_in_ifaddrhead))->sin_addr; IN_IFADDR_RUNLOCK(&in_ifa_tracker); if (cred != NULL && (error = prison_get_ip4(cred, &faddr)) != 0) return (error); } else if (faddr.s_addr == (u_long)INADDR_BROADCAST) { IN_IFADDR_RLOCK(&in_ifa_tracker); - if (TAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags & + if (CK_STAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags & IFF_BROADCAST) - faddr = satosin(&TAILQ_FIRST( + faddr = satosin(&CK_STAILQ_FIRST( &V_in_ifaddrhead)->ia_broadaddr)->sin_addr; IN_IFADDR_RUNLOCK(&in_ifa_tracker); } @@ -1104,7 +1107,7 @@ in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam, if (imo->imo_multicast_ifp != NULL) { ifp = imo->imo_multicast_ifp; IN_IFADDR_RLOCK(&in_ifa_tracker); - TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { + CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { if ((ia->ia_ifp == ifp) && (cred == NULL || prison_check_ip4(cred, @@ -1236,9 +1239,28 @@ in_pcbrele_rlocked(struct inpcb *inp) } return (0); } - + KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__)); - +#ifdef TCPHPTS + if (inp->inp_in_hpts || inp->inp_in_input) { + struct tcp_hpts_entry *hpts; + /* + * We should not be on the hpts at + * this point in any form. we must + * get the lock to be sure. + */ + hpts = tcp_hpts_lock(inp); + if (inp->inp_in_hpts) + panic("Hpts:%p inp:%p at free still on hpts", + hpts, inp); + mtx_unlock(&hpts->p_mtx); + hpts = tcp_input_lock(inp); + if (inp->inp_in_input) + panic("Hpts:%p inp:%p at free still on input hpts", + hpts, inp); + mtx_unlock(&hpts->p_mtx); + } +#endif INP_RUNLOCK(inp); pcbinfo = inp->inp_pcbinfo; uma_zfree(pcbinfo->ipi_zone, inp); @@ -1267,7 +1289,26 @@ in_pcbrele_wlocked(struct inpcb *inp) } KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__)); - +#ifdef TCPHPTS + if (inp->inp_in_hpts || inp->inp_in_input) { + struct tcp_hpts_entry *hpts; + /* + * We should not be on the hpts at + * this point in any form. we must + * get the lock to be sure. + */ + hpts = tcp_hpts_lock(inp); + if (inp->inp_in_hpts) + panic("Hpts:%p inp:%p at free still on hpts", + hpts, inp); + mtx_unlock(&hpts->p_mtx); + hpts = tcp_input_lock(inp); + if (inp->inp_in_input) + panic("Hpts:%p inp:%p at free still on input hpts", + hpts, inp); + mtx_unlock(&hpts->p_mtx); + } +#endif INP_WUNLOCK(inp); pcbinfo = inp->inp_pcbinfo; uma_zfree(pcbinfo->ipi_zone, inp); @@ -1284,6 +1325,28 @@ in_pcbrele(struct inpcb *inp) return (in_pcbrele_wlocked(inp)); } +void +in_pcblist_rele_rlocked(epoch_context_t ctx) +{ + struct in_pcblist *il; + struct inpcb *inp; + struct inpcbinfo *pcbinfo; + int i, n; + + il = __containerof(ctx, struct in_pcblist, il_epoch_ctx); + pcbinfo = il->il_pcbinfo; + n = il->il_count; + INP_INFO_WLOCK(pcbinfo); + for (i = 0; i < n; i++) { + inp = il->il_inp_list[i]; + INP_RLOCK(inp); + if (!in_pcbrele_rlocked(inp)) + INP_RUNLOCK(inp); + } + INP_INFO_WUNLOCK(pcbinfo); + free(il, M_TEMP); +} + /* * Unconditionally schedule an inpcb to be freed by decrementing its * reference count, which should occur only after the inpcb has been detached @@ -1298,8 +1361,21 @@ in_pcbfree(struct inpcb *inp) { struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; +#ifdef INET6 + struct ip6_moptions *im6o = NULL; +#endif +#ifdef INET + struct ip_moptions *imo = NULL; +#endif KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__)); + KASSERT((inp->inp_flags2 & INP_FREED) == 0, + ("%s: called twice for pcb %p", __func__, inp)); + if (inp->inp_flags2 & INP_FREED) { + INP_WUNLOCK(inp); + return; + } + #ifdef INVARIANTS if (pcbinfo == &V_tcbinfo) { INP_INFO_LOCK_ASSERT(pcbinfo); @@ -1309,6 +1385,10 @@ in_pcbfree(struct inpcb *inp) #endif INP_WLOCK_ASSERT(inp); +#ifdef INET + imo = inp->inp_moptions; + inp->inp_moptions = NULL; +#endif /* XXXRW: Do as much as possible here. */ #if defined(IPSEC) || defined(IPSEC_SUPPORT) if (inp->inp_sp != NULL) @@ -1321,16 +1401,12 @@ in_pcbfree(struct inpcb *inp) #ifdef INET6 if (inp->inp_vflag & INP_IPV6PROTO) { ip6_freepcbopts(inp->in6p_outputopts); - if (inp->in6p_moptions != NULL) - ip6_freemoptions(inp->in6p_moptions); + im6o = inp->in6p_moptions; + inp->in6p_moptions = NULL; } #endif if (inp->inp_options) (void)m_free(inp->inp_options); -#ifdef INET - if (inp->inp_moptions != NULL) - inp_freemoptions(inp->inp_moptions); -#endif RO_INVALIDATE_CACHE(&inp->inp_route); inp->inp_vflag = 0; @@ -1339,6 +1415,12 @@ in_pcbfree(struct inpcb *inp) #ifdef MAC mac_inpcb_destroy(inp); #endif +#ifdef INET6 + ip6_freemoptions(im6o); +#endif +#ifdef INET + inp_freemoptions(imo); +#endif if (!in_pcbrele_wlocked(inp)) INP_WUNLOCK(inp); } @@ -1492,11 +1574,14 @@ in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp) /* * Drop multicast group membership if we joined * through the interface being detached. + * + * XXX This can all be deferred to an epoch_call */ for (i = 0, gap = 0; i < imo->imo_num_memberships; i++) { if (imo->imo_membership[i]->inm_ifp == ifp) { - in_delmulti(imo->imo_membership[i]); + IN_MULTI_LOCK_ASSERT(); + in_leavegroup_locked(imo->imo_membership[i], NULL); gap++; } else if (gap != 0) imo->imo_membership[i - gap] = diff --git a/freebsd/sys/netinet/in_pcb.h b/freebsd/sys/netinet/in_pcb.h index 574ab407..d00dd456 100644 --- a/freebsd/sys/netinet/in_pcb.h +++ b/freebsd/sys/netinet/in_pcb.h @@ -41,6 +41,7 @@ #define _NETINET_IN_PCB_H_ #include <sys/queue.h> +#include <sys/epoch.h> #include <sys/_lock.h> #include <sys/_mutex.h> #include <sys/_rwlock.h> @@ -156,6 +157,7 @@ struct in_conninfo { * from the global list. * * Key: + * (b) - Protected by the hpts lock. * (c) - Constant after initialization * (g) - Protected by the pcbgroup lock * (i) - Protected by the inpcb lock @@ -164,6 +166,51 @@ struct in_conninfo { * (h) - Protected by the pcbhash lock for the inpcb * (s) - Protected by another subsystem's locks * (x) - Undefined locking + * + * Notes on the tcp_hpts: + * + * First Hpts lock order is + * 1) INP_WLOCK() + * 2) HPTS_LOCK() i.e. hpts->pmtx + * + * To insert a TCB on the hpts you *must* be holding the INP_WLOCK(). + * You may check the inp->inp_in_hpts flag without the hpts lock. + * The hpts is the only one that will clear this flag holding + * only the hpts lock. This means that in your tcp_output() + * routine when you test for the inp_in_hpts flag to be 1 + * it may be transitioning to 0 (by the hpts). + * That's ok since that will just mean an extra call to tcp_output + * that most likely will find the call you executed + * (when the mis-match occured) will have put the TCB back + * on the hpts and it will return. If your + * call did not add the inp back to the hpts then you will either + * over-send or the cwnd will block you from sending more. + * + * Note you should also be holding the INP_WLOCK() when you + * call the remove from the hpts as well. Though usually + * you are either doing this from a timer, where you need and have + * the INP_WLOCK() or from destroying your TCB where again + * you should already have the INP_WLOCK(). + * + * The inp_hpts_cpu, inp_hpts_cpu_set, inp_input_cpu and + * inp_input_cpu_set fields are controlled completely by + * the hpts. Do not ever set these. The inp_hpts_cpu_set + * and inp_input_cpu_set fields indicate if the hpts has + * setup the respective cpu field. It is advised if this + * field is 0, to enqueue the packet with the appropriate + * hpts_immediate() call. If the _set field is 1, then + * you may compare the inp_*_cpu field to the curcpu and + * may want to again insert onto the hpts if these fields + * are not equal (i.e. you are not on the expected CPU). + * + * A note on inp_hpts_calls and inp_input_calls, these + * flags are set when the hpts calls either the output + * or do_segment routines respectively. If the routine + * being called wants to use this, then it needs to + * clear the flag before returning. The hpts will not + * clear the flag. The flags can be used to tell if + * the hpts is the function calling the respective + * routine. * * A few other notes: * @@ -190,14 +237,45 @@ struct inpcb { LIST_ENTRY(inpcb) inp_pcbgrouphash; /* (g/i) hash list */ struct rwlock inp_lock; /* Cache line #2 (amd64) */ -#define inp_start_zero inp_refcount +#define inp_start_zero inp_hpts #define inp_zero_size (sizeof(struct inpcb) - \ offsetof(struct inpcb, inp_start_zero)) + TAILQ_ENTRY(inpcb) inp_hpts; /* pacing out queue next lock(b) */ + + uint32_t inp_hpts_request; /* Current hpts request, zero if + * fits in the pacing window (i&b). */ + /* + * Note the next fields are protected by a + * different lock (hpts-lock). This means that + * they must correspond in size to the smallest + * protectable bit field (uint8_t on x86, and + * other platfomrs potentially uint32_t?). Also + * since CPU switches can occur at different times the two + * fields can *not* be collapsed into a signal bit field. + */ +#if defined(__amd64__) || defined(__i386__) + volatile uint8_t inp_in_hpts; /* on output hpts (lock b) */ + volatile uint8_t inp_in_input; /* on input hpts (lock b) */ +#else + volatile uint32_t inp_in_hpts; /* on output hpts (lock b) */ + volatile uint32_t inp_in_input; /* on input hpts (lock b) */ +#endif + volatile uint16_t inp_hpts_cpu; /* Lock (i) */ u_int inp_refcount; /* (i) refcount */ int inp_flags; /* (i) generic IP/datagram flags */ int inp_flags2; /* (i) generic IP/datagram flags #2*/ + volatile uint16_t inp_input_cpu; /* Lock (i) */ + volatile uint8_t inp_hpts_cpu_set :1, /* on output hpts (i) */ + inp_input_cpu_set : 1, /* on input hpts (i) */ + inp_hpts_calls :1, /* (i) from output hpts */ + inp_input_calls :1, /* (i) from input hpts */ + inp_spare_bits2 : 4; + uint8_t inp_spare_byte; /* Compiler hole */ void *inp_ppcb; /* (i) pointer to per-protocol pcb */ struct socket *inp_socket; /* (i) back pointer to socket */ + uint32_t inp_hptsslot; /* Hpts wheel slot this tcb is Lock(i&b) */ + uint32_t inp_hpts_drop_reas; /* reason we are dropping the PCB (lock i&b) */ + TAILQ_ENTRY(inpcb) inp_input; /* pacing in queue next lock(b) */ struct inpcbinfo *inp_pcbinfo; /* (c) PCB list info */ struct inpcbgroup *inp_pcbgroup; /* (g/i) PCB group list */ LIST_ENTRY(inpcb) inp_pcbgroup_wild; /* (g/i/h) group wildcard entry */ @@ -330,6 +408,13 @@ struct inpcbport { u_short phd_port; }; +struct in_pcblist { + int il_count; + struct epoch_context il_epoch_ctx; + struct inpcbinfo *il_pcbinfo; + struct inpcb *il_inp_list[0]; +}; + /*- * Global data structure for each high-level protocol (UDP, TCP, ...) in both * IPv4 and IPv6. Holds inpcb lists and information for managing them. @@ -638,6 +723,7 @@ short inp_so_options(const struct inpcb *inp); #define INP_RECVRSSBUCKETID 0x00000200 /* populate recv datagram with bucket id */ #define INP_RATE_LIMIT_CHANGED 0x00000400 /* rate limit needs attention */ #define INP_ORIGDSTADDR 0x00000800 /* receive IP dst address/port */ +#define INP_CANNOT_DO_ECN 0x00001000 /* The stack does not do ECN */ /* * Flags passed to in_pcblookup*() functions. @@ -751,6 +837,7 @@ void in_pcbrehash_mbuf(struct inpcb *, struct mbuf *); int in_pcbrele(struct inpcb *); int in_pcbrele_rlocked(struct inpcb *); int in_pcbrele_wlocked(struct inpcb *); +void in_pcblist_rele_rlocked(epoch_context_t ctx); void in_losing(struct inpcb *); void in_pcbsetsolabel(struct socket *so); int in_getpeeraddr(struct socket *so, struct sockaddr **nam); diff --git a/freebsd/sys/netinet/in_proto.c b/freebsd/sys/netinet/in_proto.c index f1dec6c5..a563c950 100644 --- a/freebsd/sys/netinet/in_proto.c +++ b/freebsd/sys/netinet/in_proto.c @@ -229,7 +229,6 @@ struct protosw inetsw[] = { .pr_flags = PR_ATOMIC|PR_ADDR|PR_LASTHDR, .pr_input = encap4_input, .pr_ctloutput = rip_ctloutput, - .pr_init = encap_init, .pr_usrreqs = &rip_usrreqs }, { @@ -239,7 +238,6 @@ struct protosw inetsw[] = { .pr_flags = PR_ATOMIC|PR_ADDR|PR_LASTHDR, .pr_input = encap4_input, .pr_ctloutput = rip_ctloutput, - .pr_init = encap_init, .pr_usrreqs = &rip_usrreqs }, { @@ -249,7 +247,6 @@ struct protosw inetsw[] = { .pr_flags = PR_ATOMIC|PR_ADDR|PR_LASTHDR, .pr_input = encap4_input, .pr_ctloutput = rip_ctloutput, - .pr_init = encap_init, .pr_usrreqs = &rip_usrreqs }, { @@ -259,7 +256,6 @@ struct protosw inetsw[] = { .pr_flags = PR_ATOMIC|PR_ADDR|PR_LASTHDR, .pr_input = encap4_input, .pr_ctloutput = rip_ctloutput, - .pr_init = encap_init, .pr_usrreqs = &rip_usrreqs }, # ifdef INET6 @@ -270,7 +266,6 @@ struct protosw inetsw[] = { .pr_flags = PR_ATOMIC|PR_ADDR|PR_LASTHDR, .pr_input = encap4_input, .pr_ctloutput = rip_ctloutput, - .pr_init = encap_init, .pr_usrreqs = &rip_usrreqs }, #endif diff --git a/freebsd/sys/netinet/in_var.h b/freebsd/sys/netinet/in_var.h index ff722fc9..5b7a464b 100644 --- a/freebsd/sys/netinet/in_var.h +++ b/freebsd/sys/netinet/in_var.h @@ -55,6 +55,7 @@ struct in_aliasreq { struct igmp_ifsoftc; struct in_multi; struct lltable; +SLIST_HEAD(in_multi_head, in_multi); /* * IPv4 per-interface state. @@ -79,7 +80,7 @@ struct in_ifaddr { u_long ia_subnet; /* subnet address */ u_long ia_subnetmask; /* mask of subnet */ LIST_ENTRY(in_ifaddr) ia_hash; /* entry in bucket of inet addresses */ - TAILQ_ENTRY(in_ifaddr) ia_link; /* list of internet addresses */ + CK_STAILQ_ENTRY(in_ifaddr) ia_link; /* list of internet addresses */ struct sockaddr_in ia_addr; /* reserve space for interface name */ struct sockaddr_in ia_dstaddr; /* reserve space for broadcast addr */ #define ia_broadaddr ia_dstaddr @@ -106,7 +107,7 @@ extern u_char inetctlerrmap[]; /* * Hash table for IP addresses. */ -TAILQ_HEAD(in_ifaddrhead, in_ifaddr); +CK_STAILQ_HEAD(in_ifaddrhead, in_ifaddr); LIST_HEAD(in_ifaddrhashhead, in_ifaddr); VNET_DECLARE(struct in_ifaddrhashhead *, in_ifaddrhashtbl); @@ -171,12 +172,10 @@ do { \ /* struct rm_priotracker *t; */ \ do { \ IN_IFADDR_RLOCK((t)); \ - for ((ia) = TAILQ_FIRST(&V_in_ifaddrhead); \ + for ((ia) = CK_STAILQ_FIRST(&V_in_ifaddrhead); \ (ia) != NULL && (ia)->ia_ifp != (ifp); \ - (ia) = TAILQ_NEXT((ia), ia_link)) \ + (ia) = CK_STAILQ_NEXT((ia), ia_link)) \ continue; \ - if ((ia) != NULL) \ - ifa_ref(&(ia)->ia_ifa); \ IN_IFADDR_RUNLOCK((t)); \ } while (0) @@ -329,21 +328,53 @@ SYSCTL_DECL(_net_inet_raw); * consumers of IN_*_MULTI() macros should acquire the locks before * calling them; users of the in_{add,del}multi() functions should not. */ -extern struct mtx in_multi_mtx; -#define IN_MULTI_LOCK() mtx_lock(&in_multi_mtx) -#define IN_MULTI_UNLOCK() mtx_unlock(&in_multi_mtx) -#define IN_MULTI_LOCK_ASSERT() mtx_assert(&in_multi_mtx, MA_OWNED) -#define IN_MULTI_UNLOCK_ASSERT() mtx_assert(&in_multi_mtx, MA_NOTOWNED) +extern struct mtx in_multi_list_mtx; +extern struct sx in_multi_sx; + +#define IN_MULTI_LIST_LOCK() mtx_lock(&in_multi_list_mtx) +#define IN_MULTI_LIST_UNLOCK() mtx_unlock(&in_multi_list_mtx) +#define IN_MULTI_LIST_LOCK_ASSERT() mtx_assert(&in_multi_list_mtx, MA_OWNED) +#define IN_MULTI_LIST_UNLOCK_ASSERT() mtx_assert(&in_multi_list_mtx, MA_NOTOWNED) + +#define IN_MULTI_LOCK() sx_xlock(&in_multi_sx) +#define IN_MULTI_UNLOCK() sx_xunlock(&in_multi_sx) +#define IN_MULTI_LOCK_ASSERT() sx_assert(&in_multi_sx, SA_XLOCKED) +#define IN_MULTI_UNLOCK_ASSERT() sx_assert(&in_multi_sx, SA_XUNLOCKED) + +void inm_disconnect(struct in_multi *inm); +extern int ifma_restart; /* Acquire an in_multi record. */ static __inline void inm_acquire_locked(struct in_multi *inm) { - IN_MULTI_LOCK_ASSERT(); + IN_MULTI_LIST_LOCK_ASSERT(); ++inm->inm_refcount; } +static __inline void +inm_acquire(struct in_multi *inm) +{ + IN_MULTI_LIST_LOCK(); + inm_acquire_locked(inm); + IN_MULTI_LIST_UNLOCK(); +} + +static __inline void +inm_rele_locked(struct in_multi_head *inmh, struct in_multi *inm) +{ + MPASS(inm->inm_refcount > 0); + IN_MULTI_LIST_LOCK_ASSERT(); + + if (--inm->inm_refcount == 0) { + MPASS(inmh != NULL); + inm_disconnect(inm); + inm->inm_ifma->ifma_protospec = NULL; + SLIST_INSERT_HEAD(inmh, inm, inm_nrele); + } +} + /* * Return values for imo_multi_filter(). */ @@ -364,11 +395,10 @@ void inm_commit(struct in_multi *); void inm_clear_recorded(struct in_multi *); void inm_print(const struct in_multi *); int inm_record_source(struct in_multi *inm, const in_addr_t); -void inm_release(struct in_multi *); -void inm_release_locked(struct in_multi *); +void inm_release_deferred(struct in_multi *); +void inm_release_list_deferred(struct in_multi_head *); struct in_multi * - in_addmulti(struct in_addr *, struct ifnet *); -void in_delmulti(struct in_multi *); +in_addmulti(struct in_addr *, struct ifnet *); int in_joingroup(struct ifnet *, const struct in_addr *, /*const*/ struct in_mfilter *, struct in_multi **); int in_joingroup_locked(struct ifnet *, const struct in_addr *, diff --git a/freebsd/sys/netinet/ip_carp.c b/freebsd/sys/netinet/ip_carp.c index e2bd0a0a..6f5160e0 100644 --- a/freebsd/sys/netinet/ip_carp.c +++ b/freebsd/sys/netinet/ip_carp.c @@ -57,7 +57,6 @@ __FBSDID("$FreeBSD$"); #include <sys/counter.h> #include <net/ethernet.h> -#include <net/fddi.h> #include <net/if.h> #include <net/if_var.h> #include <net/if_dl.h> @@ -213,11 +212,13 @@ static VNET_DEFINE(int, carp_senderr_adj) = CARP_MAXSKEW; static VNET_DEFINE(int, carp_ifdown_adj) = CARP_MAXSKEW; #define V_carp_ifdown_adj VNET(carp_ifdown_adj) +static int carp_allow_sysctl(SYSCTL_HANDLER_ARGS); static int carp_demote_adj_sysctl(SYSCTL_HANDLER_ARGS); SYSCTL_NODE(_net_inet, IPPROTO_CARP, carp, CTLFLAG_RW, 0, "CARP"); -SYSCTL_INT(_net_inet_carp, OID_AUTO, allow, CTLFLAG_VNET | CTLFLAG_RW, - &VNET_NAME(carp_allow), 0, "Accept incoming CARP packets"); +SYSCTL_PROC(_net_inet_carp, OID_AUTO, allow, + CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW, 0, 0, carp_allow_sysctl, "I", + "Accept incoming CARP packets"); SYSCTL_INT(_net_inet_carp, OID_AUTO, preempt, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(carp_preempt), 0, "High-priority backup preemption mode"); SYSCTL_INT(_net_inet_carp, OID_AUTO, log, CTLFLAG_VNET | CTLFLAG_RW, @@ -277,8 +278,7 @@ SYSCTL_VNET_PCPUSTAT(_net_inet_carp, OID_AUTO, stats, struct carpstats, } while (0) #define IFNET_FOREACH_IFA(ifp, ifa) \ - IF_ADDR_LOCK_ASSERT(ifp); \ - TAILQ_FOREACH((ifa), &(ifp)->if_addrhead, ifa_link) \ + CK_STAILQ_FOREACH((ifa), &(ifp)->if_addrhead, ifa_link) \ if ((ifa)->ifa_carp != NULL) #define CARP_FOREACH_IFA(sc, ifa) \ @@ -879,7 +879,7 @@ carp_best_ifa(int af, struct ifnet *ifp) return (NULL); best = NULL; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family == af && (best == NULL || ifa_preferred(best, ifa))) best = ifa; @@ -1161,7 +1161,7 @@ carp_iamatch6(struct ifnet *ifp, struct in6_addr *taddr) ifa = NULL; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; if (!IN6_ARE_ADDR_EQUAL(taddr, IFA_IN6(ifa))) @@ -1294,7 +1294,8 @@ carp_setrun(struct carp_softc *sc, sa_family_t af) if ((sc->sc_carpdev->if_flags & IFF_UP) == 0 || sc->sc_carpdev->if_link_state != LINK_STATE_UP || - (sc->sc_naddrs == 0 && sc->sc_naddrs6 == 0)) + (sc->sc_naddrs == 0 && sc->sc_naddrs6 == 0) || + !V_carp_allow) return; switch (sc->sc_state) { @@ -1408,7 +1409,7 @@ carp_multicast_setup(struct carp_if *cif, sa_family_t sa) break; } in6m = NULL; - if ((error = in6_mc_join(ifp, &in6, NULL, &in6m, 0)) != 0) { + if ((error = in6_joingroup(ifp, &in6, NULL, &in6m, 0)) != 0) { free(im6o->im6o_membership, M_CARP); break; } @@ -1423,13 +1424,13 @@ carp_multicast_setup(struct carp_if *cif, sa_family_t sa) in6.s6_addr32[3] = 0; in6.s6_addr8[12] = 0xff; if ((error = in6_setscope(&in6, ifp, NULL)) != 0) { - in6_mc_leave(im6o->im6o_membership[0], NULL); + in6_leavegroup(im6o->im6o_membership[0], NULL); free(im6o->im6o_membership, M_CARP); break; } in6m = NULL; - if ((error = in6_mc_join(ifp, &in6, NULL, &in6m, 0)) != 0) { - in6_mc_leave(im6o->im6o_membership[0], NULL); + if ((error = in6_joingroup(ifp, &in6, NULL, &in6m, 0)) != 0) { + in6_leavegroup(im6o->im6o_membership[0], NULL); free(im6o->im6o_membership, M_CARP); break; } @@ -1472,8 +1473,8 @@ carp_multicast_cleanup(struct carp_if *cif, sa_family_t sa) if (cif->cif_naddrs6 == 0) { struct ip6_moptions *im6o = &cif->cif_im6o; - in6_mc_leave(im6o->im6o_membership[0], NULL); - in6_mc_leave(im6o->im6o_membership[1], NULL); + in6_leavegroup(im6o->im6o_membership[0], NULL); + in6_leavegroup(im6o->im6o_membership[1], NULL); KASSERT(im6o->im6o_mfilters == NULL, ("%s: im6o_mfilters != NULL", __func__)); free(im6o->im6o_membership, M_CARP); @@ -1528,18 +1529,6 @@ carp_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *sa) eh->ether_shost[5] = sc->sc_vhid; } break; - case IFT_FDDI: { - struct fddi_header *fh; - - fh = mtod(m, struct fddi_header *); - fh->fddi_shost[0] = 0; - fh->fddi_shost[1] = 0; - fh->fddi_shost[2] = 0x5e; - fh->fddi_shost[3] = 0; - fh->fddi_shost[4] = 1; - fh->fddi_shost[5] = sc->sc_vhid; - } - break; default: printf("%s: carp is not supported for the %d interface type\n", ifp->if_xname, ifp->if_type); @@ -1721,7 +1710,6 @@ carp_ioctl(struct ifreq *ifr, u_long cmd, struct thread *td) case IFT_ETHER: case IFT_L2VLAN: case IFT_BRIDGE: - case IFT_FDDI: break; default: error = EOPNOTSUPP; @@ -2057,7 +2045,8 @@ carp_sc_state(struct carp_softc *sc) CARP_LOCK_ASSERT(sc); if (sc->sc_carpdev->if_link_state != LINK_STATE_UP || - !(sc->sc_carpdev->if_flags & IFF_UP)) { + !(sc->sc_carpdev->if_flags & IFF_UP) || + !V_carp_allow) { callout_stop(&sc->sc_ad_tmo); #ifdef INET callout_stop(&sc->sc_md_tmo); @@ -2088,6 +2077,33 @@ carp_demote_adj(int adj, char *reason) } static int +carp_allow_sysctl(SYSCTL_HANDLER_ARGS) +{ + int new, error; + struct carp_softc *sc; + + new = V_carp_allow; + error = sysctl_handle_int(oidp, &new, 0, req); + if (error || !req->newptr) + return (error); + + if (V_carp_allow != new) { + V_carp_allow = new; + + mtx_lock(&carp_mtx); + LIST_FOREACH(sc, &carp_list, sc_next) { + CARP_LOCK(sc); + if (curvnet == sc->sc_carpdev->if_vnet) + carp_sc_state(sc); + CARP_UNLOCK(sc); + } + mtx_unlock(&carp_mtx); + } + + return (0); +} + +static int carp_demote_adj_sysctl(SYSCTL_HANDLER_ARGS) { int new, error; diff --git a/freebsd/sys/netinet/ip_divert.c b/freebsd/sys/netinet/ip_divert.c index 53a0445e..84f39023 100644 --- a/freebsd/sys/netinet/ip_divert.c +++ b/freebsd/sys/netinet/ip_divert.c @@ -76,7 +76,6 @@ __FBSDID("$FreeBSD$"); #endif #include <security/mac/mac_framework.h> - /* * Divert sockets */ @@ -237,7 +236,7 @@ divert_packet(struct mbuf *m, int incoming) /* Find IP address for receive interface */ ifp = m->m_pkthdr.rcvif; if_addr_rlock(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; divsrc.sin_addr = @@ -471,13 +470,15 @@ div_output(struct socket *so, struct mbuf *m, struct sockaddr_in *sin, bzero(sin->sin_zero, sizeof(sin->sin_zero)); sin->sin_port = 0; + NET_EPOCH_ENTER(); ifa = ifa_ifwithaddr((struct sockaddr *) sin); if (ifa == NULL) { error = EADDRNOTAVAIL; + NET_EPOCH_EXIT(); goto cantsend; } m->m_pkthdr.rcvif = ifa->ifa_ifp; - ifa_free(ifa); + NET_EPOCH_EXIT(); } #ifdef MAC mac_socket_create_mbuf(so, m); @@ -553,6 +554,7 @@ div_detach(struct socket *so) KASSERT(inp != NULL, ("div_detach: inp == NULL")); INP_INFO_WLOCK(&V_divcbinfo); INP_WLOCK(inp); + /* XXX defer destruction to epoch_call */ in_pcbdetach(inp); in_pcbfree(inp); INP_INFO_WUNLOCK(&V_divcbinfo); @@ -632,6 +634,7 @@ static int div_pcblist(SYSCTL_HANDLER_ARGS) { int error, i, n; + struct in_pcblist *il; struct inpcb *inp, **inp_list; inp_gen_t gencnt; struct xinpgen xig; @@ -671,9 +674,8 @@ div_pcblist(SYSCTL_HANDLER_ARGS) if (error) return error; - inp_list = malloc(n * sizeof *inp_list, M_TEMP, M_WAITOK); - if (inp_list == NULL) - return ENOMEM; + il = malloc(sizeof(struct in_pcblist) + n * sizeof(struct inpcb *), M_TEMP, M_WAITOK|M_ZERO_INVARIANTS); + inp_list = il->il_inp_list; INP_INFO_RLOCK(&V_divcbinfo); for (inp = LIST_FIRST(V_divcbinfo.ipi_listhead), i = 0; inp && i < n; @@ -702,14 +704,9 @@ div_pcblist(SYSCTL_HANDLER_ARGS) } else INP_RUNLOCK(inp); } - INP_INFO_WLOCK(&V_divcbinfo); - for (i = 0; i < n; i++) { - inp = inp_list[i]; - INP_RLOCK(inp); - if (!in_pcbrele_rlocked(inp)) - INP_RUNLOCK(inp); - } - INP_INFO_WUNLOCK(&V_divcbinfo); + il->il_count = n; + il->il_pcbinfo = &V_divcbinfo; + epoch_call(net_epoch_preempt, &il->il_epoch_ctx, in_pcblist_rele_rlocked); if (!error) { /* @@ -726,7 +723,6 @@ div_pcblist(SYSCTL_HANDLER_ARGS) INP_INFO_RUNLOCK(&V_divcbinfo); error = SYSCTL_OUT(req, &xig, sizeof xig); } - free(inp_list, M_TEMP); return error; } @@ -806,6 +802,7 @@ div_modevent(module_t mod, int type, void *unused) break; } ip_divert_ptr = NULL; + /* XXX defer to epoch_call ? */ err = pf_proto_unregister(PF_INET, IPPROTO_DIVERT, SOCK_RAW); INP_INFO_WUNLOCK(&V_divcbinfo); #ifndef VIMAGE diff --git a/freebsd/sys/netinet/ip_encap.c b/freebsd/sys/netinet/ip_encap.c index d0866b00..52cd0b40 100644 --- a/freebsd/sys/netinet/ip_encap.c +++ b/freebsd/sys/netinet/ip_encap.c @@ -110,15 +110,6 @@ static struct mtx encapmtx; MTX_SYSINIT(encapmtx, &encapmtx, "encapmtx", MTX_DEF); static LIST_HEAD(, encaptab) encaptab = LIST_HEAD_INITIALIZER(encaptab); -/* - * We currently keey encap_init() for source code compatibility reasons -- - * it's referenced by KAME pieces in netinet6. - */ -void -encap_init(void) -{ -} - #ifdef INET int encap4_input(struct mbuf **mp, int *offp, int proto) diff --git a/freebsd/sys/netinet/ip_encap.h b/freebsd/sys/netinet/ip_encap.h index bbbee390..ef232189 100644 --- a/freebsd/sys/netinet/ip_encap.h +++ b/freebsd/sys/netinet/ip_encap.h @@ -50,7 +50,6 @@ struct encaptab { void *arg; /* passed via m->m_pkthdr.aux */ }; -void encap_init(void); int encap4_input(struct mbuf **, int *, int); int encap6_input(struct mbuf **, int *, int); const struct encaptab *encap_attach(int, int, const struct sockaddr *, diff --git a/freebsd/sys/netinet/ip_icmp.c b/freebsd/sys/netinet/ip_icmp.c index b03fea56..3fc59a14 100644 --- a/freebsd/sys/netinet/ip_icmp.c +++ b/freebsd/sys/netinet/ip_icmp.c @@ -71,6 +71,7 @@ __FBSDID("$FreeBSD$"); #include <netinet/tcpip.h> #include <netinet/icmp_var.h> + #ifdef INET #include <machine/in_cksum.h> @@ -407,6 +408,7 @@ icmp_input(struct mbuf **mp, int *offp, int proto) inet_ntoa_r(ip->ip_dst, dstbuf), icmplen); } #endif + NET_EPOCH_ENTER(); if (icmplen < ICMP_MINLEN) { ICMPSTAT_INC(icps_tooshort); goto freeit; @@ -414,6 +416,7 @@ icmp_input(struct mbuf **mp, int *offp, int proto) i = hlen + min(icmplen, ICMP_ADVLENMIN); if (m->m_len < i && (m = m_pullup(m, i)) == NULL) { ICMPSTAT_INC(icps_tooshort); + NET_EPOCH_EXIT(); return (IPPROTO_DONE); } ip = mtod(m, struct ip *); @@ -531,6 +534,7 @@ icmp_input(struct mbuf **mp, int *offp, int proto) if (m->m_len < i && (m = m_pullup(m, i)) == NULL) { /* This should actually not happen */ ICMPSTAT_INC(icps_tooshort); + NET_EPOCH_EXIT(); return (IPPROTO_DONE); } ip = mtod(m, struct ip *); @@ -606,10 +610,8 @@ icmp_input(struct mbuf **mp, int *offp, int proto) (struct sockaddr *)&icmpdst, m->m_pkthdr.rcvif); if (ia == NULL) break; - if (ia->ia_ifp == NULL) { - ifa_free(&ia->ia_ifa); + if (ia->ia_ifp == NULL) break; - } icp->icmp_type = ICMP_MASKREPLY; if (V_icmpmaskfake == 0) icp->icmp_mask = ia->ia_sockmask.sin_addr.s_addr; @@ -621,11 +623,11 @@ icmp_input(struct mbuf **mp, int *offp, int proto) else if (ia->ia_ifp->if_flags & IFF_POINTOPOINT) ip->ip_src = satosin(&ia->ia_dstaddr)->sin_addr; } - ifa_free(&ia->ia_ifa); reflect: ICMPSTAT_INC(icps_reflect); ICMPSTAT_INC(icps_outhist[icp->icmp_type]); icmp_reflect(m); + NET_EPOCH_EXIT(); return (IPPROTO_DONE); case ICMP_REDIRECT: @@ -702,11 +704,13 @@ reflect: } raw: + NET_EPOCH_EXIT(); *mp = m; rip_input(mp, offp, proto); return (IPPROTO_DONE); freeit: + NET_EPOCH_EXIT(); m_freem(m); return (IPPROTO_DONE); } @@ -762,7 +766,7 @@ icmp_reflect(struct mbuf *m) ifp = m->m_pkthdr.rcvif; if (ifp != NULL && ifp->if_flags & IFF_BROADCAST) { IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; ia = ifatoia(ifa); @@ -783,7 +787,7 @@ icmp_reflect(struct mbuf *m) */ if (V_icmp_rfi && ifp != NULL) { IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; ia = ifatoia(ifa); @@ -801,7 +805,7 @@ icmp_reflect(struct mbuf *m) */ if (V_reply_src[0] != '\0' && (ifp = ifunit(V_reply_src))) { IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; ia = ifatoia(ifa); diff --git a/freebsd/sys/netinet/ip_input.c b/freebsd/sys/netinet/ip_input.c index 2c8bf427..343eec5e 100644 --- a/freebsd/sys/netinet/ip_input.c +++ b/freebsd/sys/netinet/ip_input.c @@ -306,7 +306,7 @@ ip_init(void) struct protosw *pr; int i; - TAILQ_INIT(&V_in_ifaddrhead); + CK_STAILQ_INIT(&V_in_ifaddrhead); V_in_ifaddrhashtbl = hashinit(INADDR_NHASH, M_IFADDR, &V_in_ifaddrhmask); /* Initialize IP reassembly queue. */ @@ -401,7 +401,7 @@ ip_destroy(void *unused __unused) /* Make sure the IPv4 routes are gone as well. */ IFNET_RLOCK(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) rt_flushifroutes_af(ifp, AF_INET); IFNET_RUNLOCK(); @@ -652,7 +652,7 @@ passin: * we receive might be for us (and let the upper layers deal * with it). */ - if (TAILQ_EMPTY(&V_in_ifaddrhead) && + if (CK_STAILQ_EMPTY(&V_in_ifaddrhead) && (m->m_flags & (M_MCAST|M_BCAST)) == 0) goto ours; @@ -709,7 +709,7 @@ passin: */ if (ifp != NULL && ifp->if_flags & IFF_BROADCAST) { IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; ia = ifatoia(ifa); @@ -979,9 +979,9 @@ ip_forward(struct mbuf *m, int srcrt) #else in_rtalloc_ign(&ro, 0, M_GETFIB(m)); #endif + NET_EPOCH_ENTER(); if (ro.ro_rt != NULL) { ia = ifatoia(ro.ro_rt->rt_ifa); - ifa_ref(&ia->ia_ifa); } else ia = NULL; /* @@ -1027,7 +1027,7 @@ ip_forward(struct mbuf *m, int srcrt) m_freem(mcopy); if (error != EINPROGRESS) IPSTAT_INC(ips_cantforward); - return; + goto out; } /* No IPsec processing required */ } @@ -1080,16 +1080,12 @@ ip_forward(struct mbuf *m, int srcrt) else { if (mcopy) m_freem(mcopy); - if (ia != NULL) - ifa_free(&ia->ia_ifa); - return; + goto out; } } - if (mcopy == NULL) { - if (ia != NULL) - ifa_free(&ia->ia_ifa); - return; - } + if (mcopy == NULL) + goto out; + switch (error) { @@ -1131,13 +1127,11 @@ ip_forward(struct mbuf *m, int srcrt) case ENOBUFS: case EACCES: /* ipfw denied packet */ m_freem(mcopy); - if (ia != NULL) - ifa_free(&ia->ia_ifa); - return; + goto out; } - if (ia != NULL) - ifa_free(&ia->ia_ifa); icmp_error(mcopy, type, code, dest.s_addr, mtu); + out: + NET_EPOCH_EXIT(); } #define CHECK_SO_CT(sp, ct) \ diff --git a/freebsd/sys/netinet/ip_mroute.c b/freebsd/sys/netinet/ip_mroute.c index 3bf4fa91..ac901601 100644 --- a/freebsd/sys/netinet/ip_mroute.c +++ b/freebsd/sys/netinet/ip_mroute.c @@ -880,13 +880,15 @@ add_vif(struct vifctl *vifcp) ifp = NULL; } else { sin.sin_addr = vifcp->vifc_lcl_addr; + NET_EPOCH_ENTER(); ifa = ifa_ifwithaddr((struct sockaddr *)&sin); if (ifa == NULL) { + NET_EPOCH_EXIT(); VIF_UNLOCK(); return EADDRNOTAVAIL; } ifp = ifa->ifa_ifp; - ifa_free(ifa); + NET_EPOCH_EXIT(); } if ((vifcp->vifc_flags & VIFF_TUNNEL) != 0) { @@ -1682,7 +1684,7 @@ send_packet(struct vif *vifp, struct mbuf *m) { struct ip_moptions imo; struct in_multi *imm[2]; - int error; + int error __unused; VIF_LOCK_ASSERT(); diff --git a/freebsd/sys/netinet/ip_options.c b/freebsd/sys/netinet/ip_options.c index d85aecf3..cc2f3eed 100644 --- a/freebsd/sys/netinet/ip_options.c +++ b/freebsd/sys/netinet/ip_options.c @@ -112,6 +112,7 @@ ip_dooptions(struct mbuf *m, int pass) struct nhop4_extended nh_ext; struct sockaddr_in ipaddr = { sizeof(ipaddr), AF_INET }; + NET_EPOCH_ENTER(); /* Ignore or reject packets with IP options. */ if (V_ip_doopts == 0) return 0; @@ -226,6 +227,7 @@ dropit: #endif IPSTAT_INC(ips_cantforward); m_freem(m); + NET_EPOCH_EXIT(); return (1); } } @@ -252,7 +254,6 @@ dropit: memcpy(cp + off, &(IA_SIN(ia)->sin_addr), sizeof(struct in_addr)); - ifa_free(&ia->ia_ifa); } else { /* XXX MRT 0 for routing */ if (fib4_lookup_nh_ext(M_GETFIB(m), @@ -300,7 +301,6 @@ dropit: if ((ia = (INA)ifa_ifwithaddr((SA)&ipaddr)) != NULL) { memcpy(cp + off, &(IA_SIN(ia)->sin_addr), sizeof(struct in_addr)); - ifa_free(&ia->ia_ifa); } else if (fib4_lookup_nh_ext(M_GETFIB(m), ipaddr.sin_addr, 0, 0, &nh_ext) == 0) { memcpy(cp + off, &nh_ext.nh_src, @@ -355,7 +355,6 @@ dropit: continue; (void)memcpy(sin, &IA_SIN(ia)->sin_addr, sizeof(struct in_addr)); - ifa_free(&ia->ia_ifa); cp[IPOPT_OFFSET] += sizeof(struct in_addr); off += sizeof(struct in_addr); break; @@ -383,12 +382,14 @@ dropit: cp[IPOPT_OFFSET] += sizeof(uint32_t); } } + NET_EPOCH_EXIT(); if (forward && V_ipforwarding) { ip_forward(m, 1); return (1); } return (0); bad: + NET_EPOCH_EXIT(); icmp_error(m, type, code, 0, 0); IPSTAT_INC(ips_badoptions); return (1); diff --git a/freebsd/sys/netinet/ip_output.c b/freebsd/sys/netinet/ip_output.c index 21b3919a..792f2311 100644 --- a/freebsd/sys/netinet/ip_output.c +++ b/freebsd/sys/netinet/ip_output.c @@ -227,7 +227,6 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags, struct route iproute; struct rtentry *rte; /* cache for ro->ro_rt */ uint32_t fibnum; - int have_ia_ref; #if defined(IPSEC) || defined(IPSEC_SUPPORT) int no_route_but_check_spd = 0; #endif @@ -283,6 +282,7 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags, dst->sin_len = sizeof(*dst); dst->sin_addr = ip->ip_dst; } + NET_EPOCH_ENTER(); again: /* * Validate route against routing table additions; @@ -308,7 +308,6 @@ again: rte = NULL; } ia = NULL; - have_ia_ref = 0; /* * If routing to interface only, short circuit routing lookup. * The use of an all-ones broadcast address implies this; an @@ -324,7 +323,6 @@ again: error = ENETUNREACH; goto bad; } - have_ia_ref = 1; ip->ip_dst.s_addr = INADDR_BROADCAST; dst->sin_addr = ip->ip_dst; ifp = ia->ia_ifp; @@ -339,7 +337,6 @@ again: error = ENETUNREACH; goto bad; } - have_ia_ref = 1; ifp = ia->ia_ifp; ip->ip_ttl = 1; isbroadcast = ifp->if_flags & IFF_BROADCAST ? @@ -352,8 +349,6 @@ again: */ ifp = imo->imo_multicast_ifp; IFP_TO_IA(ifp, ia, &in_ifa_tracker); - if (ia) - have_ia_ref = 1; isbroadcast = 0; /* fool gcc */ } else { /* @@ -581,8 +576,6 @@ sendit: case -1: /* Need to try again */ /* Reset everything for a new round */ RO_RTFREE(ro); - if (have_ia_ref) - ifa_free(&ia->ia_ifa); ro->ro_prepend = NULL; rte = NULL; gw = dst; @@ -737,10 +730,9 @@ done: * calling RTFREE on it again. */ ro->ro_rt = NULL; - if (have_ia_ref) - ifa_free(&ia->ia_ifa); + NET_EPOCH_EXIT(); return (error); -bad: + bad: m_freem(m); goto done; } diff --git a/freebsd/sys/netinet/ip_var.h b/freebsd/sys/netinet/ip_var.h index 9e7ee591..f874628a 100644 --- a/freebsd/sys/netinet/ip_var.h +++ b/freebsd/sys/netinet/ip_var.h @@ -36,6 +36,7 @@ #define _NETINET_IP_VAR_H_ #include <sys/queue.h> +#include <sys/epoch.h> /* * Overlay for ip header used by other protocols (tcp, udp). @@ -95,7 +96,7 @@ struct ip_moptions { u_short imo_max_memberships; /* max memberships this socket */ struct in_multi **imo_membership; /* group memberships */ struct in_mfilter *imo_mfilters; /* source filters */ - STAILQ_ENTRY(ip_moptions) imo_link; + struct epoch_context imo_epoch_ctx; }; struct ipstat { @@ -175,6 +176,7 @@ struct ip; struct inpcb; struct route; struct sockopt; +struct inpcbinfo; VNET_DECLARE(int, ip_defttl); /* default IP ttl */ VNET_DECLARE(int, ipforwarding); /* ip forwarding */ diff --git a/freebsd/sys/netinet/netdump/netdump.h b/freebsd/sys/netinet/netdump/netdump.h new file mode 100644 index 00000000..12a527ee --- /dev/null +++ b/freebsd/sys/netinet/netdump/netdump.h @@ -0,0 +1,132 @@ +/*- + * Copyright (c) 2005-2014 Sandvine Incorporated + * Copyright (c) 2000 Darrell Anderson <anderson@cs.duke.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _NETINET_NETDUMP_H_ +#define _NETINET_NETDUMP_H_ + +#include <sys/types.h> +#include <sys/disk.h> +#include <sys/ioccom.h> + +#include <net/if.h> +#include <netinet/in.h> + +#define NETDUMP_PORT 20023 /* Server UDP port for heralds. */ +#define NETDUMP_ACKPORT 20024 /* Client UDP port for acks. */ + +#define NETDUMP_HERALD 1 /* Broadcast before starting a dump. */ +#define NETDUMP_FINISHED 2 /* Send after finishing a dump. */ +#define NETDUMP_VMCORE 3 /* Contains dump data. */ +#define NETDUMP_KDH 4 /* Contains kernel dump header. */ +#define NETDUMP_EKCD_KEY 5 /* Contains kernel dump key. */ + +#define NETDUMP_DATASIZE 4096 /* Arbitrary packet size limit. */ + +struct netdump_msg_hdr { + uint32_t mh_type; /* Netdump message type. */ + uint32_t mh_seqno; /* Match acks with msgs. */ + uint64_t mh_offset; /* vmcore offset (bytes). */ + uint32_t mh_len; /* Attached data (bytes). */ + uint32_t mh__pad; +} __packed; + +struct netdump_ack { + uint32_t na_seqno; /* Match acks with msgs. */ +} __packed; + +struct netdump_conf { +#ifndef __rtems__ + struct diocskerneldump_arg ndc_kda; +#endif /* __rtems__ */ + char ndc_iface[IFNAMSIZ]; + struct in_addr ndc_server; + struct in_addr ndc_client; + struct in_addr ndc_gateway; +}; + +#define _PATH_NETDUMP "/dev/netdump" + +#define NETDUMPGCONF _IOR('n', 1, struct netdump_conf) +#define NETDUMPSCONF _IOW('n', 2, struct netdump_conf) + +#ifdef _KERNEL +#ifdef NETDUMP + +#define NETDUMP_MAX_IN_FLIGHT 64 + +enum netdump_ev { + NETDUMP_START, + NETDUMP_END, +}; + +struct ifnet; +struct mbuf; + +void netdump_reinit(struct ifnet *); + +typedef void netdump_init_t(struct ifnet *, int *nrxr, int *ncl, int *clsize); +typedef void netdump_event_t(struct ifnet *, enum netdump_ev); +typedef int netdump_transmit_t(struct ifnet *, struct mbuf *); +typedef int netdump_poll_t(struct ifnet *, int); + +struct netdump_methods { + netdump_init_t *nd_init; + netdump_event_t *nd_event; + netdump_transmit_t *nd_transmit; + netdump_poll_t *nd_poll; +}; + +#define NETDUMP_DEFINE(driver) \ + static netdump_init_t driver##_netdump_init; \ + static netdump_event_t driver##_netdump_event; \ + static netdump_transmit_t driver##_netdump_transmit; \ + static netdump_poll_t driver##_netdump_poll; \ + \ + static struct netdump_methods driver##_netdump_methods = { \ + .nd_init = driver##_netdump_init, \ + .nd_event = driver##_netdump_event, \ + .nd_transmit = driver##_netdump_transmit, \ + .nd_poll = driver##_netdump_poll, \ + } + +#define NETDUMP_REINIT(ifp) netdump_reinit(ifp) + +#define NETDUMP_SET(ifp, driver) \ + (ifp)->if_netdump_methods = &driver##_netdump_methods + +#else /* !NETDUMP */ + +#define NETDUMP_DEFINE(driver) +#define NETDUMP_REINIT(ifp) +#define NETDUMP_SET(ifp, driver) + +#endif /* NETDUMP */ +#endif /* _KERNEL */ + +#endif /* _NETINET_NETDUMP_H_ */ diff --git a/freebsd/sys/netinet/raw_ip.c b/freebsd/sys/netinet/raw_ip.c index 0ed185ae..7dea3ec1 100644 --- a/freebsd/sys/netinet/raw_ip.c +++ b/freebsd/sys/netinet/raw_ip.c @@ -745,7 +745,7 @@ rip_ctlinput(int cmd, struct sockaddr *sa, void *vip) switch (cmd) { case PRC_IFDOWN: IN_IFADDR_RLOCK(&in_ifa_tracker); - TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { + CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { if (ia->ia_ifa.ifa_addr == sa && (ia->ia_flags & IFA_ROUTE)) { ifa_ref(&ia->ia_ifa); @@ -771,7 +771,7 @@ rip_ctlinput(int cmd, struct sockaddr *sa, void *vip) case PRC_IFUP: IN_IFADDR_RLOCK(&in_ifa_tracker); - TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { + CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { if (ia->ia_ifa.ifa_addr == sa) break; } @@ -853,6 +853,7 @@ rip_detach(struct socket *so) ip_rsvp_force_done(so); if (so == V_ip_rsvpd) ip_rsvp_done(); + /* XXX defer to epoch_call */ in_pcbdetach(inp); in_pcbfree(inp); INP_INFO_WUNLOCK(&V_ripcbinfo); @@ -930,7 +931,7 @@ rip_bind(struct socket *so, struct sockaddr *nam, struct thread *td) inp = sotoinpcb(so); KASSERT(inp != NULL, ("rip_bind: inp == NULL")); - if (TAILQ_EMPTY(&V_ifnet) || + if (CK_STAILQ_EMPTY(&V_ifnet) || (addr->sin_family != AF_INET && addr->sin_family != AF_IMPLINK) || (addr->sin_addr.s_addr && (inp->inp_flags & INP_BINDANY) == 0 && @@ -955,7 +956,7 @@ rip_connect(struct socket *so, struct sockaddr *nam, struct thread *td) if (nam->sa_len != sizeof(*addr)) return (EINVAL); - if (TAILQ_EMPTY(&V_ifnet)) + if (CK_STAILQ_EMPTY(&V_ifnet)) return (EADDRNOTAVAIL); if (addr->sin_family != AF_INET && addr->sin_family != AF_IMPLINK) return (EAFNOSUPPORT); @@ -1022,6 +1023,7 @@ static int rip_pcblist(SYSCTL_HANDLER_ARGS) { int error, i, n; + struct in_pcblist *il; struct inpcb *inp, **inp_list; inp_gen_t gencnt; struct xinpgen xig; @@ -1056,9 +1058,8 @@ rip_pcblist(SYSCTL_HANDLER_ARGS) if (error) return (error); - inp_list = malloc(n * sizeof *inp_list, M_TEMP, M_WAITOK); - if (inp_list == NULL) - return (ENOMEM); + il = malloc(sizeof(struct in_pcblist) + n * sizeof(struct inpcb *), M_TEMP, M_WAITOK|M_ZERO_INVARIANTS); + inp_list = il->il_inp_list; INP_INFO_RLOCK(&V_ripcbinfo); for (inp = LIST_FIRST(V_ripcbinfo.ipi_listhead), i = 0; inp && i < n; @@ -1087,14 +1088,9 @@ rip_pcblist(SYSCTL_HANDLER_ARGS) } else INP_RUNLOCK(inp); } - INP_INFO_WLOCK(&V_ripcbinfo); - for (i = 0; i < n; i++) { - inp = inp_list[i]; - INP_RLOCK(inp); - if (!in_pcbrele_rlocked(inp)) - INP_RUNLOCK(inp); - } - INP_INFO_WUNLOCK(&V_ripcbinfo); + il->il_count = n; + il->il_pcbinfo = &V_ripcbinfo; + epoch_call(net_epoch_preempt, &il->il_epoch_ctx, in_pcblist_rele_rlocked); if (!error) { /* @@ -1110,7 +1106,6 @@ rip_pcblist(SYSCTL_HANDLER_ARGS) INP_INFO_RUNLOCK(&V_ripcbinfo); error = SYSCTL_OUT(req, &xig, sizeof xig); } - free(inp_list, M_TEMP); return (error); } diff --git a/freebsd/sys/netinet/sctp_bsd_addr.c b/freebsd/sys/netinet/sctp_bsd_addr.c index 7e2ef189..94c23bff 100644 --- a/freebsd/sys/netinet/sctp_bsd_addr.c +++ b/freebsd/sys/netinet/sctp_bsd_addr.c @@ -209,13 +209,13 @@ sctp_init_ifns_for_vrf(int vrfid) #endif IFNET_RLOCK(); - TAILQ_FOREACH(ifn, &MODULE_GLOBAL(ifnet), if_link) { + CK_STAILQ_FOREACH(ifn, &MODULE_GLOBAL(ifnet), if_link) { if (sctp_is_desired_interface_type(ifn) == 0) { /* non desired type */ continue; } IF_ADDR_RLOCK(ifn); - TAILQ_FOREACH(ifa, &ifn->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifn->if_addrhead, ifa_link) { if (ifa->ifa_addr == NULL) { continue; } @@ -362,11 +362,11 @@ void struct ifaddr *ifa; IFNET_RLOCK(); - TAILQ_FOREACH(ifn, &MODULE_GLOBAL(ifnet), if_link) { + CK_STAILQ_FOREACH(ifn, &MODULE_GLOBAL(ifnet), if_link) { if (!(*pred) (ifn)) { continue; } - TAILQ_FOREACH(ifa, &ifn->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifn->if_addrhead, ifa_link) { sctp_addr_change(ifa, add ? RTM_ADD : RTM_DELETE); } } @@ -389,10 +389,7 @@ sctp_get_mbuf_for_msg(unsigned int space_needed, int want_header, m_freem(m); return (NULL); } - } - if (SCTP_BUF_NEXT(m)) { - sctp_m_freem(SCTP_BUF_NEXT(m)); - SCTP_BUF_NEXT(m) = NULL; + KASSERT(SCTP_BUF_NEXT(m) == NULL, ("%s: no chain allowed", __FUNCTION__)); } #ifdef SCTP_MBUF_LOGGING if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_MBUF_LOGGING_ENABLE) { diff --git a/freebsd/sys/netinet/sctp_indata.c b/freebsd/sys/netinet/sctp_indata.c index 3325dd03..98b397a2 100644 --- a/freebsd/sys/netinet/sctp_indata.c +++ b/freebsd/sys/netinet/sctp_indata.c @@ -1673,9 +1673,7 @@ sctp_process_a_data_chunk(struct sctp_tcb *stcb, struct sctp_association *asoc, struct sctp_nets *net, uint32_t *high_tsn, int *abort_flag, int *break_flag, int last_chunk, uint8_t chk_type) { - /* Process a data chunk */ - /* struct sctp_tmit_chunk *chk; */ - struct sctp_tmit_chunk *chk; + struct sctp_tmit_chunk *chk = NULL; /* make gcc happy */ uint32_t tsn, fsn, gap, mid; struct mbuf *dmbuf; int the_len; @@ -3623,7 +3621,9 @@ sctp_strike_gap_ack_chunks(struct sctp_tcb *stcb, struct sctp_association *asoc, SCTP_SO_NOT_LOCKED); } /* Make sure to flag we had a FR */ - tp1->whoTo->net_ack++; + if (tp1->whoTo != NULL) { + tp1->whoTo->net_ack++; + } continue; } } diff --git a/freebsd/sys/netinet/sctp_input.c b/freebsd/sys/netinet/sctp_input.c index 9a74ef4b..ee206551 100644 --- a/freebsd/sys/netinet/sctp_input.c +++ b/freebsd/sys/netinet/sctp_input.c @@ -2618,7 +2618,7 @@ sctp_handle_cookie_echo(struct mbuf *m, int iphlen, int offset, (sizeof(uint32_t)))); diff = now; timevalsub(&diff, &time_expires); - if (diff.tv_sec > UINT32_MAX / 1000000) { + if ((uint32_t)diff.tv_sec > UINT32_MAX / 1000000) { staleness = UINT32_MAX; } else { staleness = diff.tv_sec * 1000000; diff --git a/freebsd/sys/netinet/sctp_os_bsd.h b/freebsd/sys/netinet/sctp_os_bsd.h index c9eaa069..d8d9e6e8 100644 --- a/freebsd/sys/netinet/sctp_os_bsd.h +++ b/freebsd/sys/netinet/sctp_os_bsd.h @@ -40,7 +40,6 @@ __FBSDID("$FreeBSD$"); /* * includes */ -#include <rtems/bsd/local/opt_compat.h> #include <rtems/bsd/local/opt_inet6.h> #include <rtems/bsd/local/opt_inet.h> #include <rtems/bsd/local/opt_sctp.h> diff --git a/freebsd/sys/netinet/sctp_output.c b/freebsd/sys/netinet/sctp_output.c index 9dd2e0fa..bdef958c 100644 --- a/freebsd/sys/netinet/sctp_output.c +++ b/freebsd/sys/netinet/sctp_output.c @@ -7452,7 +7452,7 @@ dont_do_it: /* Not enough room for a chunk header, get some */ struct mbuf *m; - m = sctp_get_mbuf_for_msg(1, 0, M_NOWAIT, 0, MT_DATA); + m = sctp_get_mbuf_for_msg(1, 0, M_NOWAIT, 1, MT_DATA); if (m == NULL) { /* * we're in trouble here. _PREPEND below will free @@ -11032,9 +11032,8 @@ sctp_send_resp_msg(struct sockaddr *src, struct sockaddr *dst, struct sctp_chunkhdr *ch; #if defined(INET) || defined(INET6) struct udphdr *udp; - int ret; #endif - int len, cause_len, padding_len; + int ret, len, cause_len, padding_len; #ifdef INET struct sockaddr_in *src_sin, *dst_sin; struct ip *ip; @@ -11261,9 +11260,13 @@ sctp_send_resp_msg(struct sockaddr *src, struct sockaddr *dst, SCTP_LTRACE_ERR_RET_PKT(mout, NULL, NULL, NULL, SCTP_FROM_SCTP_OUTPUT, EFAULT); return; } + SCTPDBG(SCTP_DEBUG_OUTPUT3, "return from send is %d\n", ret); SCTP_STAT_INCR(sctps_sendpackets); SCTP_STAT_INCR_COUNTER64(sctps_outpackets); SCTP_STAT_INCR_COUNTER64(sctps_outcontrolchunks); + if (ret) { + SCTP_STAT_INCR(sctps_senderrors); + } return; } diff --git a/freebsd/sys/netinet/sctp_usrreq.c b/freebsd/sys/netinet/sctp_usrreq.c index 05ddee01..071d44c2 100644 --- a/freebsd/sys/netinet/sctp_usrreq.c +++ b/freebsd/sys/netinet/sctp_usrreq.c @@ -206,7 +206,7 @@ sctp_notify(struct sctp_inpcb *inp, #endif /* no need to unlock here, since the TCB is gone */ } else if (icmp_code == ICMP_UNREACH_NEEDFRAG) { - if ((net->dest_state & SCTP_ADDR_NO_PMTUD) == 0) { + if (net->dest_state & SCTP_ADDR_NO_PMTUD) { SCTP_TCB_UNLOCK(stcb); return; } @@ -707,22 +707,10 @@ sctp_disconnect(struct socket *so) if (SCTP_GET_STATE(asoc) != SCTP_STATE_COOKIE_WAIT) { /* Left with Data unread */ - struct mbuf *err; - - err = sctp_get_mbuf_for_msg(sizeof(struct sctp_paramhdr), 0, M_NOWAIT, 1, MT_DATA); - if (err) { - /* - * Fill in the user - * initiated abort - */ - struct sctp_paramhdr *ph; + struct mbuf *op_err; - ph = mtod(err, struct sctp_paramhdr *); - SCTP_BUF_LEN(err) = sizeof(struct sctp_paramhdr); - ph->param_type = htons(SCTP_CAUSE_USER_INITIATED_ABT); - ph->param_length = htons(SCTP_BUF_LEN(err)); - } - sctp_send_abort_tcb(stcb, err, SCTP_SO_LOCKED); + op_err = sctp_generate_cause(SCTP_CAUSE_USER_INITIATED_ABT, ""); + sctp_send_abort_tcb(stcb, op_err, SCTP_SO_LOCKED); SCTP_STAT_INCR_COUNTER32(sctps_aborted); } SCTP_INP_RUNLOCK(inp); diff --git a/freebsd/sys/netinet/sctputil.c b/freebsd/sys/netinet/sctputil.c index 5511df64..aad1e19d 100644 --- a/freebsd/sys/netinet/sctputil.c +++ b/freebsd/sys/netinet/sctputil.c @@ -74,6 +74,7 @@ extern const struct sctp_ss_functions sctp_ss_functions[]; void sctp_sblog(struct sockbuf *sb, struct sctp_tcb *stcb, int from, int incr) { +#if defined(SCTP_LOCAL_TRACE_BUF) struct sctp_cwnd_log sctp_clog; sctp_clog.x.sb.stcb = stcb; @@ -90,11 +91,13 @@ sctp_sblog(struct sockbuf *sb, struct sctp_tcb *stcb, int from, int incr) sctp_clog.x.misc.log2, sctp_clog.x.misc.log3, sctp_clog.x.misc.log4); +#endif } void sctp_log_closing(struct sctp_inpcb *inp, struct sctp_tcb *stcb, int16_t loc) { +#if defined(SCTP_LOCAL_TRACE_BUF) struct sctp_cwnd_log sctp_clog; sctp_clog.x.close.inp = (void *)inp; @@ -114,11 +117,13 @@ sctp_log_closing(struct sctp_inpcb *inp, struct sctp_tcb *stcb, int16_t loc) sctp_clog.x.misc.log2, sctp_clog.x.misc.log3, sctp_clog.x.misc.log4); +#endif } void rto_logging(struct sctp_nets *net, int from) { +#if defined(SCTP_LOCAL_TRACE_BUF) struct sctp_cwnd_log sctp_clog; memset(&sctp_clog, 0, sizeof(sctp_clog)); @@ -131,11 +136,13 @@ rto_logging(struct sctp_nets *net, int from) sctp_clog.x.misc.log2, sctp_clog.x.misc.log3, sctp_clog.x.misc.log4); +#endif } void sctp_log_strm_del_alt(struct sctp_tcb *stcb, uint32_t tsn, uint16_t sseq, uint16_t stream, int from) { +#if defined(SCTP_LOCAL_TRACE_BUF) struct sctp_cwnd_log sctp_clog; sctp_clog.x.strlog.stcb = stcb; @@ -151,11 +158,13 @@ sctp_log_strm_del_alt(struct sctp_tcb *stcb, uint32_t tsn, uint16_t sseq, uint16 sctp_clog.x.misc.log2, sctp_clog.x.misc.log3, sctp_clog.x.misc.log4); +#endif } void sctp_log_nagle_event(struct sctp_tcb *stcb, int action) { +#if defined(SCTP_LOCAL_TRACE_BUF) struct sctp_cwnd_log sctp_clog; sctp_clog.x.nagle.stcb = (void *)stcb; @@ -170,11 +179,13 @@ sctp_log_nagle_event(struct sctp_tcb *stcb, int action) sctp_clog.x.misc.log2, sctp_clog.x.misc.log3, sctp_clog.x.misc.log4); +#endif } void sctp_log_sack(uint32_t old_cumack, uint32_t cumack, uint32_t tsn, uint16_t gaps, uint16_t dups, int from) { +#if defined(SCTP_LOCAL_TRACE_BUF) struct sctp_cwnd_log sctp_clog; sctp_clog.x.sack.cumack = cumack; @@ -189,11 +200,13 @@ sctp_log_sack(uint32_t old_cumack, uint32_t cumack, uint32_t tsn, uint16_t gaps, sctp_clog.x.misc.log2, sctp_clog.x.misc.log3, sctp_clog.x.misc.log4); +#endif } void sctp_log_map(uint32_t map, uint32_t cum, uint32_t high, int from) { +#if defined(SCTP_LOCAL_TRACE_BUF) struct sctp_cwnd_log sctp_clog; memset(&sctp_clog, 0, sizeof(sctp_clog)); @@ -207,11 +220,13 @@ sctp_log_map(uint32_t map, uint32_t cum, uint32_t high, int from) sctp_clog.x.misc.log2, sctp_clog.x.misc.log3, sctp_clog.x.misc.log4); +#endif } void sctp_log_fr(uint32_t biggest_tsn, uint32_t biggest_new_tsn, uint32_t tsn, int from) { +#if defined(SCTP_LOCAL_TRACE_BUF) struct sctp_cwnd_log sctp_clog; memset(&sctp_clog, 0, sizeof(sctp_clog)); @@ -225,12 +240,14 @@ sctp_log_fr(uint32_t biggest_tsn, uint32_t biggest_new_tsn, uint32_t tsn, int fr sctp_clog.x.misc.log2, sctp_clog.x.misc.log3, sctp_clog.x.misc.log4); +#endif } #ifdef SCTP_MBUF_LOGGING void sctp_log_mb(struct mbuf *m, int from) { +#if defined(SCTP_LOCAL_TRACE_BUF) struct sctp_cwnd_log sctp_clog; sctp_clog.x.mb.mp = m; @@ -251,6 +268,7 @@ sctp_log_mb(struct mbuf *m, int from) sctp_clog.x.misc.log2, sctp_clog.x.misc.log3, sctp_clog.x.misc.log4); +#endif } void @@ -267,6 +285,7 @@ sctp_log_mbc(struct mbuf *m, int from) void sctp_log_strm_del(struct sctp_queued_to_read *control, struct sctp_queued_to_read *poschk, int from) { +#if defined(SCTP_LOCAL_TRACE_BUF) struct sctp_cwnd_log sctp_clog; if (control == NULL) { @@ -291,11 +310,13 @@ sctp_log_strm_del(struct sctp_queued_to_read *control, struct sctp_queued_to_rea sctp_clog.x.misc.log2, sctp_clog.x.misc.log3, sctp_clog.x.misc.log4); +#endif } void sctp_log_cwnd(struct sctp_tcb *stcb, struct sctp_nets *net, int augment, uint8_t from) { +#if defined(SCTP_LOCAL_TRACE_BUF) struct sctp_cwnd_log sctp_clog; sctp_clog.x.cwnd.net = net; @@ -326,11 +347,13 @@ sctp_log_cwnd(struct sctp_tcb *stcb, struct sctp_nets *net, int augment, uint8_t sctp_clog.x.misc.log2, sctp_clog.x.misc.log3, sctp_clog.x.misc.log4); +#endif } void sctp_log_lock(struct sctp_inpcb *inp, struct sctp_tcb *stcb, uint8_t from) { +#if defined(SCTP_LOCAL_TRACE_BUF) struct sctp_cwnd_log sctp_clog; memset(&sctp_clog, 0, sizeof(sctp_clog)); @@ -370,11 +393,13 @@ sctp_log_lock(struct sctp_inpcb *inp, struct sctp_tcb *stcb, uint8_t from) sctp_clog.x.misc.log2, sctp_clog.x.misc.log3, sctp_clog.x.misc.log4); +#endif } void sctp_log_maxburst(struct sctp_tcb *stcb, struct sctp_nets *net, int error, int burst, uint8_t from) { +#if defined(SCTP_LOCAL_TRACE_BUF) struct sctp_cwnd_log sctp_clog; memset(&sctp_clog, 0, sizeof(sctp_clog)); @@ -397,11 +422,13 @@ sctp_log_maxburst(struct sctp_tcb *stcb, struct sctp_nets *net, int error, int b sctp_clog.x.misc.log2, sctp_clog.x.misc.log3, sctp_clog.x.misc.log4); +#endif } void sctp_log_rwnd(uint8_t from, uint32_t peers_rwnd, uint32_t snd_size, uint32_t overhead) { +#if defined(SCTP_LOCAL_TRACE_BUF) struct sctp_cwnd_log sctp_clog; sctp_clog.x.rwnd.rwnd = peers_rwnd; @@ -415,11 +442,13 @@ sctp_log_rwnd(uint8_t from, uint32_t peers_rwnd, uint32_t snd_size, uint32_t ove sctp_clog.x.misc.log2, sctp_clog.x.misc.log3, sctp_clog.x.misc.log4); +#endif } void sctp_log_rwnd_set(uint8_t from, uint32_t peers_rwnd, uint32_t flight_size, uint32_t overhead, uint32_t a_rwndval) { +#if defined(SCTP_LOCAL_TRACE_BUF) struct sctp_cwnd_log sctp_clog; sctp_clog.x.rwnd.rwnd = peers_rwnd; @@ -433,12 +462,14 @@ sctp_log_rwnd_set(uint8_t from, uint32_t peers_rwnd, uint32_t flight_size, uint3 sctp_clog.x.misc.log2, sctp_clog.x.misc.log3, sctp_clog.x.misc.log4); +#endif } #ifdef SCTP_MBCNT_LOGGING static void sctp_log_mbcnt(uint8_t from, uint32_t total_oq, uint32_t book, uint32_t total_mbcnt_q, uint32_t mbcnt) { +#if defined(SCTP_LOCAL_TRACE_BUF) struct sctp_cwnd_log sctp_clog; sctp_clog.x.mbcnt.total_queue_size = total_oq; @@ -452,21 +483,25 @@ sctp_log_mbcnt(uint8_t from, uint32_t total_oq, uint32_t book, uint32_t total_mb sctp_clog.x.misc.log2, sctp_clog.x.misc.log3, sctp_clog.x.misc.log4); +#endif } #endif void sctp_misc_ints(uint8_t from, uint32_t a, uint32_t b, uint32_t c, uint32_t d) { +#if defined(SCTP_LOCAL_TRACE_BUF) SCTP_CTR6(KTR_SCTP, "SCTP:%d[%d]:%x-%x-%x-%x", SCTP_LOG_MISC_EVENT, from, a, b, c, d); +#endif } void sctp_wakeup_log(struct sctp_tcb *stcb, uint32_t wake_cnt, int from) { +#if defined(SCTP_LOCAL_TRACE_BUF) struct sctp_cwnd_log sctp_clog; sctp_clog.x.wake.stcb = (void *)stcb; @@ -508,11 +543,13 @@ sctp_wakeup_log(struct sctp_tcb *stcb, uint32_t wake_cnt, int from) sctp_clog.x.misc.log2, sctp_clog.x.misc.log3, sctp_clog.x.misc.log4); +#endif } void sctp_log_block(uint8_t from, struct sctp_association *asoc, size_t sendlen) { +#if defined(SCTP_LOCAL_TRACE_BUF) struct sctp_cwnd_log sctp_clog; sctp_clog.x.blk.onsb = asoc->total_output_queue_size; @@ -529,6 +566,7 @@ sctp_log_block(uint8_t from, struct sctp_association *asoc, size_t sendlen) sctp_clog.x.misc.log2, sctp_clog.x.misc.log3, sctp_clog.x.misc.log4); +#endif } int @@ -760,8 +798,8 @@ sctp_stop_timers_for_shutdown(struct sctp_tcb *stcb) } /* - * a list of sizes based on typical mtu's, used only if next hop size not - * returned. + * A list of sizes based on typical mtu's, used only if next hop size not + * returned. These values MUST be multiples of 4 and MUST be ordered. */ static uint32_t sctp_mtu_sizes[] = { 68, @@ -770,29 +808,32 @@ static uint32_t sctp_mtu_sizes[] = { 512, 544, 576, - 1006, + 1004, 1492, 1500, 1536, - 2002, + 2000, 2048, 4352, 4464, 8166, - 17914, + 17912, 32000, - 65535 + 65532 }; /* - * Return the largest MTU smaller than val. If there is no - * entry, just return val. + * Return the largest MTU in sctp_mtu_sizes smaller than val. + * If val is smaller than the minimum, just return the largest + * multiple of 4 smaller or equal to val. + * Ensure that the result is a multiple of 4. */ uint32_t sctp_get_prev_mtu(uint32_t val) { uint32_t i; + val &= 0xfffffffc; if (val <= sctp_mtu_sizes[0]) { return (val); } @@ -801,12 +842,16 @@ sctp_get_prev_mtu(uint32_t val) break; } } + KASSERT((sctp_mtu_sizes[i - 1] & 0x00000003) == 0, + ("sctp_mtu_sizes[%u] not a multiple of 4", i - 1)); return (sctp_mtu_sizes[i - 1]); } /* - * Return the smallest MTU larger than val. If there is no - * entry, just return val. + * Return the smallest MTU in sctp_mtu_sizes larger than val. + * If val is larger than the maximum, just return the largest multiple of 4 smaller + * or equal to val. + * Ensure that the result is a multiple of 4. */ uint32_t sctp_get_next_mtu(uint32_t val) @@ -814,8 +859,11 @@ sctp_get_next_mtu(uint32_t val) /* select another MTU that is just bigger than this one */ uint32_t i; + val &= 0xfffffffc; for (i = 0; i < (sizeof(sctp_mtu_sizes) / sizeof(uint32_t)); i++) { if (val < sctp_mtu_sizes[i]) { + KASSERT((sctp_mtu_sizes[i] & 0x00000003) == 0, + ("sctp_mtu_sizes[%u] not a multiple of 4", i)); return (sctp_mtu_sizes[i]); } } @@ -2662,6 +2710,13 @@ sctp_notify_assoc_change(uint16_t state, struct sctp_tcb *stcb, notif_len = (unsigned int)sizeof(struct sctp_assoc_change); if (abort != NULL) { abort_len = ntohs(abort->ch.chunk_length); + /* + * Only SCTP_CHUNK_BUFFER_SIZE are guaranteed to be + * contiguous. + */ + if (abort_len > SCTP_CHUNK_BUFFER_SIZE) { + abort_len = SCTP_CHUNK_BUFFER_SIZE; + } } else { abort_len = 0; } @@ -3567,6 +3622,13 @@ sctp_notify_remote_error(struct sctp_tcb *stcb, uint16_t error, struct sctp_erro } if (chunk != NULL) { chunk_len = ntohs(chunk->ch.chunk_length); + /* + * Only SCTP_CHUNK_BUFFER_SIZE are guaranteed to be + * contiguous. + */ + if (chunk_len > SCTP_CHUNK_BUFFER_SIZE) { + chunk_len = SCTP_CHUNK_BUFFER_SIZE; + } } else { chunk_len = 0; } diff --git a/freebsd/sys/netinet/tcp_hpts.h b/freebsd/sys/netinet/tcp_hpts.h new file mode 100644 index 00000000..c52a1d78 --- /dev/null +++ b/freebsd/sys/netinet/tcp_hpts.h @@ -0,0 +1,304 @@ +/*- + * Copyright (c) 2016-2018 Netflix Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __tcp_hpts_h__ +#define __tcp_hpts_h__ + +/* + * The hpts uses a 102400 wheel. The wheel + * defines the time in 10 usec increments (102400 x 10). + * This gives a range of 10usec - 1024ms to place + * an entry within. If the user requests more than + * 1.024 second, a remaineder is attached and the hpts + * when seeing the remainder will re-insert the + * inpcb forward in time from where it is until + * the remainder is zero. + */ + +#define NUM_OF_HPTSI_SLOTS 102400 + +TAILQ_HEAD(hptsh, inpcb); + +/* Number of useconds in a hpts tick */ +#define HPTS_TICKS_PER_USEC 10 +#define HPTS_MS_TO_SLOTS(x) (x * 100) +#define HPTS_USEC_TO_SLOTS(x) ((x+9) /10) +#define HPTS_USEC_IN_SEC 1000000 +#define HPTS_MSEC_IN_SEC 1000 +#define HPTS_USEC_IN_MSEC 1000 + +#define DEFAULT_HPTS_LOG 3072 + +/* + * Log flags consist of + * 7f 7f 1 1 bits + * p_cpu | p_num | INPUT_ACTIVE | HPTS_ACTIVE + * + * So for example cpu 10, number 10 would with + * input active would show up as: + * p_flags = 0001010 0001010 1 0 + * <or> + * p_flags = 0x142a + */ +#define HPTS_HPTS_ACTIVE 0x01 +#define HPTS_INPUT_ACTIVE 0x02 + +#define HPTSLOG_IMMEDIATE 1 +#define HPTSLOG_INSERT_NORMAL 2 +#define HPTSLOG_INSERT_SLEEPER 3 +#define HPTSLOG_SLEEP_AFTER 4 +#define HPTSLOG_SLEEP_BEFORE 5 +#define HPTSLOG_INSERTED 6 +#define HPTSLOG_WAKEUP_HPTS 7 +#define HPTSLOG_SETTORUN 8 +#define HPTSLOG_HPTSI 9 +#define HPTSLOG_TOLONG 10 +#define HPTSLOG_AWAKENS 11 +#define HPTSLOG_TIMESOUT 12 +#define HPTSLOG_SLEEPSET 13 +#define HPTSLOG_WAKEUP_INPUT 14 +#define HPTSLOG_RESCHEDULE 15 +#define HPTSLOG_AWAKE 16 +#define HPTSLOG_INP_DONE 17 + +struct hpts_log { + struct inpcb *inp; + int32_t event; + uint32_t cts; + int32_t line; + uint32_t ticknow; + uint32_t t_paceslot; + uint32_t t_hptsreq; + uint32_t p_curtick; + uint32_t p_prevtick; + uint32_t slot_req; + uint32_t p_on_queue_cnt; + uint32_t p_nxt_slot; + uint32_t p_cur_slot; + uint32_t p_hpts_sleep_time; + uint16_t p_flags; + uint8_t p_onhpts; + uint8_t p_oninput; + uint8_t is_notempty; +}; + +struct hpts_diag { + uint32_t p_hpts_active; + uint32_t p_nxt_slot; + uint32_t p_cur_slot; + uint32_t slot_req; + uint32_t inp_hptsslot; + uint32_t slot_now; + uint32_t have_slept; + uint32_t hpts_sleep_time; + uint32_t yet_to_sleep; + uint32_t need_new_to; + int32_t co_ret; + uint8_t p_on_min_sleep; +}; + +#ifdef _KERNEL +/* Each hpts has its own p_mtx which is used for locking */ +struct tcp_hpts_entry { + /* Cache line 0x00 */ + struct mtx p_mtx; /* Mutex for hpts */ + uint32_t p_hpts_active; /* Flag that says hpts is awake */ + uint32_t p_curtick; /* Current tick in 10 us the hpts is at */ + uint32_t p_prevtick; /* Previous tick in 10 us the hpts ran */ + uint32_t p_cur_slot; /* Current slot in wheel hpts is draining */ + uint32_t p_nxt_slot; /* The next slot outside the current range of + * slots that the hpts is running on. */ + int32_t p_on_queue_cnt; /* Count on queue in this hpts */ + uint32_t enobuf_cnt; + uint16_t p_log_at; + uint8_t p_direct_wake :1, /* boolean */ + p_log_wrapped :1, /* boolean */ + p_on_min_sleep:1; /* boolean */ + uint8_t p_fill; + /* Cache line 0x40 */ + void *p_inp; + struct hptsh p_input; /* For the tcp-input runner */ + /* Hptsi wheel */ + struct hptsh *p_hptss; + struct hpts_log *p_log; + uint32_t p_logsize; + int32_t p_on_inqueue_cnt; /* Count on input queue in this hpts */ + uint32_t hit_no_enobuf; + uint32_t p_dyn_adjust; + uint32_t p_hpts_sleep_time; /* Current sleep interval having a max + * of 255ms */ + uint32_t p_delayed_by; /* How much were we delayed by */ + /* Cache line 0x80 */ + struct sysctl_ctx_list hpts_ctx; + struct sysctl_oid *hpts_root; + struct intr_event *ie; + void *ie_cookie; + uint16_t p_num; /* The hpts number one per cpu */ + uint16_t p_cpu; /* The hpts CPU */ + /* There is extra space in here */ + /* Cache line 0x100 */ + struct callout co __aligned(CACHE_LINE_SIZE); +} __aligned(CACHE_LINE_SIZE); + +struct tcp_hptsi { + struct proc *rp_proc; /* Process structure for hpts */ + struct tcp_hpts_entry **rp_ent; /* Array of hptss */ + uint32_t rp_num_hptss; /* Number of hpts threads */ +}; + +#endif + +#define HPTS_REMOVE_INPUT 0x01 +#define HPTS_REMOVE_OUTPUT 0x02 +#define HPTS_REMOVE_ALL (HPTS_REMOVE_INPUT | HPTS_REMOVE_OUTPUT) + +/* + * When using the hpts, a TCP stack must make sure + * that once a INP_DROPPED flag is applied to a INP + * that it does not expect tcp_output() to ever be + * called by the hpts. The hpts will *not* call + * any output (or input) functions on a TCB that + * is in the DROPPED state. + * + * This implies final ACK's and RST's that might + * be sent when a TCB is still around must be + * sent from a routine like tcp_respond(). + */ +#define DEFAULT_MIN_SLEEP 250 /* How many usec's is default for hpts sleep + * this determines min granularity of the + * hpts. If 0, granularity is 10useconds at + * the cost of more CPU (context switching). */ +#ifdef _KERNEL +#define HPTS_MTX_ASSERT(hpts) mtx_assert(&(hpts)->p_mtx, MA_OWNED) +struct tcp_hpts_entry *tcp_hpts_lock(struct inpcb *inp); +struct tcp_hpts_entry *tcp_input_lock(struct inpcb *inp); +int __tcp_queue_to_hpts_immediate(struct inpcb *inp, int32_t line); +#define tcp_queue_to_hpts_immediate(a)__tcp_queue_to_hpts_immediate(a, __LINE__) + +struct tcp_hpts_entry *tcp_cur_hpts(struct inpcb *inp); +#define tcp_hpts_remove(a, b) __tcp_hpts_remove(a, b, __LINE__) +void __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line); + +/* + * To insert a TCB on the hpts you *must* be holding the + * INP_WLOCK(). The hpts insert code will then acqurire + * the hpts's lock and insert the TCB on the requested + * slot possibly waking up the hpts if you are requesting + * a time earlier than what the hpts is sleeping to (if + * the hpts is sleeping). You may check the inp->inp_in_hpts + * flag without the hpts lock. The hpts is the only one + * that will clear this flag holding only the hpts lock. This + * means that in your tcp_output() routine when you test for + * it to be 1 (so you wont call output) it may be transitioning + * to 0 (by the hpts). That will be fine since that will just + * mean an extra call to tcp_output that most likely will find + * the call you executed (when the mis-match occured) will have + * put the TCB back on the hpts and it will return. If your + * call did not add it back to the hpts then you will either + * over-send or the cwnd will block you from sending more. + * + * Note you should also be holding the INP_WLOCK() when you + * call the remove from the hpts as well. Thoug usually + * you are either doing this from a timer, where you need + * that INP_WLOCK() or from destroying your TCB where again + * you should already have the INP_WLOCK(). + */ +uint32_t __tcp_hpts_insert(struct inpcb *inp, uint32_t slot, int32_t line); +#define tcp_hpts_insert(a, b) __tcp_hpts_insert(a, b, __LINE__) + +uint32_t +tcp_hpts_insert_diag(struct inpcb *inp, uint32_t slot, int32_t line, struct hpts_diag *diag); + +int + __tcp_queue_to_input_locked(struct inpcb *inp, struct tcp_hpts_entry *hpts, int32_t line); +#define tcp_queue_to_input_locked(a, b) __tcp_queue_to_input_locked(a, b, __LINE__); +void +tcp_queue_pkt_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th, + int32_t tlen, int32_t drop_hdrlen, uint8_t iptos, uint8_t ti_locked); +int +__tcp_queue_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th, + int32_t tlen, int32_t drop_hdrlen, uint8_t iptos, uint8_t ti_locked, int32_t line); +#define tcp_queue_to_input(a, b, c, d, e, f, g) __tcp_queue_to_input(a, b, c, d, e, f, g, __LINE__) + +uint16_t tcp_hpts_delayedby(struct inpcb *inp); + +void __tcp_set_hpts(struct inpcb *inp, int32_t line); +#define tcp_set_hpts(a) __tcp_set_hpts(a, __LINE__) + +void __tcp_set_inp_to_drop(struct inpcb *inp, uint16_t reason, int32_t line); +#define tcp_set_inp_to_drop(a, b) __tcp_set_inp_to_drop(a, b, __LINE__) + +extern int32_t tcp_min_hptsi_time; + +static __inline uint32_t +tcp_tv_to_hptstick(struct timeval *sv) +{ + return ((sv->tv_sec * 100000) + (sv->tv_usec / 10)); +} + +static __inline uint32_t +tcp_gethptstick(struct timeval *sv) +{ + struct timeval tv; + + if (sv == NULL) + sv = &tv; + microuptime(sv); + return (tcp_tv_to_hptstick(sv)); +} + +static __inline uint32_t +tcp_tv_to_usectick(struct timeval *sv) +{ + return ((uint32_t) ((sv->tv_sec * HPTS_USEC_IN_SEC) + sv->tv_usec)); +} + +static __inline uint32_t +tcp_tv_to_mssectick(struct timeval *sv) +{ + return ((uint32_t) ((sv->tv_sec * HPTS_MSEC_IN_SEC) + (sv->tv_usec/HPTS_USEC_IN_MSEC))); +} + +static __inline void +tcp_hpts_unlock(struct tcp_hpts_entry *hpts) +{ + mtx_unlock(&hpts->p_mtx); +} + +static __inline uint32_t +tcp_get_usecs(struct timeval *tv) +{ + struct timeval tvd; + + if (tv == NULL) + tv = &tvd; + microuptime(tv); + return (tcp_tv_to_usectick(tv)); +} + +#endif /* _KERNEL */ +#endif /* __tcp_hpts_h__ */ diff --git a/freebsd/sys/netinet/tcp_input.c b/freebsd/sys/netinet/tcp_input.c index 7c907da9..20bea2de 100644 --- a/freebsd/sys/netinet/tcp_input.c +++ b/freebsd/sys/netinet/tcp_input.c @@ -1684,6 +1684,9 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, to.to_tsecr -= tp->ts_offset; if (TSTMP_GT(to.to_tsecr, tcp_ts_getticks())) to.to_tsecr = 0; + else if (tp->t_flags & TF_PREVVALID && + tp->t_badrxtwin != 0 && SEQ_LT(to.to_tsecr, tp->t_badrxtwin)) + cc_cong_signal(tp, th, CC_RTO_ERR); } /* * Process options only when we get SYN/ACK back. The SYN case @@ -1796,9 +1799,10 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, TCPSTAT_INC(tcps_predack); /* - * "bad retransmit" recovery. + * "bad retransmit" recovery without timestamps. */ - if (tp->t_rxtshift == 1 && + if ((to.to_flags & TOF_TS) == 0 && + tp->t_rxtshift == 1 && tp->t_flags & TF_PREVVALID && (int)(ticks - tp->t_badrxtwin) < 0) { cc_cong_signal(tp, th, CC_RTO_ERR); @@ -2789,8 +2793,10 @@ process_ACK: * original cwnd and ssthresh, and proceed to transmit where * we left off. */ - if (tp->t_rxtshift == 1 && tp->t_flags & TF_PREVVALID && - (int)(ticks - tp->t_badrxtwin) < 0) + if (tp->t_rxtshift == 1 && + tp->t_flags & TF_PREVVALID && + tp->t_badrxtwin && + SEQ_LT(to.to_tsecr, tp->t_badrxtwin)) cc_cong_signal(tp, th, CC_RTO_ERR); /* diff --git a/freebsd/sys/netinet/tcp_offload.c b/freebsd/sys/netinet/tcp_offload.c index 41302db1..f3ab3b50 100644 --- a/freebsd/sys/netinet/tcp_offload.c +++ b/freebsd/sys/netinet/tcp_offload.c @@ -170,6 +170,17 @@ tcp_offload_ctloutput(struct tcpcb *tp, int sopt_dir, int sopt_name) } void +tcp_offload_tcp_info(struct tcpcb *tp, struct tcp_info *ti) +{ + struct toedev *tod = tp->tod; + + KASSERT(tod != NULL, ("%s: tp->tod is NULL, tp %p", __func__, tp)); + INP_WLOCK_ASSERT(tp->t_inpcb); + + tod->tod_tcp_info(tod, tp, ti); +} + +void tcp_offload_detach(struct tcpcb *tp) { struct toedev *tod = tp->tod; diff --git a/freebsd/sys/netinet/tcp_offload.h b/freebsd/sys/netinet/tcp_offload.h index 8485fa29..f755ce7e 100644 --- a/freebsd/sys/netinet/tcp_offload.h +++ b/freebsd/sys/netinet/tcp_offload.h @@ -45,6 +45,7 @@ void tcp_offload_input(struct tcpcb *, struct mbuf *); int tcp_offload_output(struct tcpcb *); void tcp_offload_rcvd(struct tcpcb *); void tcp_offload_ctloutput(struct tcpcb *, int, int); +void tcp_offload_tcp_info(struct tcpcb *, struct tcp_info *); void tcp_offload_detach(struct tcpcb *); #endif diff --git a/freebsd/sys/netinet/tcp_output.c b/freebsd/sys/netinet/tcp_output.c index 8762407f..bdbfe984 100644 --- a/freebsd/sys/netinet/tcp_output.c +++ b/freebsd/sys/netinet/tcp_output.c @@ -208,7 +208,7 @@ tcp_output(struct tcpcb *tp) #if defined(IPSEC) || defined(IPSEC_SUPPORT) unsigned ipsec_optlen = 0; #endif - int idle, sendalot; + int idle, sendalot, curticks; int sack_rxmit, sack_bytes_rxmt; struct sackhole *p; int tso, mtu; @@ -810,9 +810,12 @@ send: /* Timestamps. */ if ((tp->t_flags & TF_RCVD_TSTMP) || ((flags & TH_SYN) && (tp->t_flags & TF_REQ_TSTMP))) { - to.to_tsval = tcp_ts_getticks() + tp->ts_offset; + curticks = tcp_ts_getticks(); + to.to_tsval = curticks + tp->ts_offset; to.to_tsecr = tp->ts_recent; to.to_flags |= TOF_TS; + if (tp->t_rxtshift == 1) + tp->t_badrxtwin = curticks; } /* Set receive buffer autosizing timestamp. */ @@ -1313,10 +1316,6 @@ send: } #endif - /* We're getting ready to send; log now. */ - TCP_LOG_EVENT(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_OUT, ERRNO_UNK, - len, NULL, false); - /* * Enable TSO and specify the size of the segments. * The TCP pseudo header checksum is always provided. @@ -1365,6 +1364,10 @@ send: #endif /* TCPDEBUG */ TCP_PROBE3(debug__output, tp, th, m); + /* We're getting ready to send; log now. */ + TCP_LOG_EVENT(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_OUT, ERRNO_UNK, + len, NULL, false); + /* * Fill in IP length and desired time to live and * send to IP level. There should be a better way @@ -1588,8 +1591,6 @@ timer: SOCKBUF_UNLOCK_ASSERT(&so->so_snd); /* Check gotos. */ switch (error) { case EACCES: - tp->t_softerror = error; - return (0); case EPERM: tp->t_softerror = error; return (error); diff --git a/freebsd/sys/netinet/tcp_seq.h b/freebsd/sys/netinet/tcp_seq.h index b29ae2aa..b6e682ec 100644 --- a/freebsd/sys/netinet/tcp_seq.h +++ b/freebsd/sys/netinet/tcp_seq.h @@ -47,10 +47,10 @@ #define SEQ_MIN(a, b) ((SEQ_LT(a, b)) ? (a) : (b)) #define SEQ_MAX(a, b) ((SEQ_GT(a, b)) ? (a) : (b)) -#define WIN_LT(a,b) ((short)(ntohs(a)-ntohs(b)) < 0) -#define WIN_LEQ(a,b) ((short)(ntohs(a)-ntohs(b)) <= 0) -#define WIN_GT(a,b) ((short)(ntohs(a)-ntohs(b)) > 0) -#define WIN_GEQ(a,b) ((short)(ntohs(a)-ntohs(b)) >= 0) +#define WIN_LT(a,b) (ntohs(a) < ntohs(b)) +#define WIN_LEQ(a,b) (ntohs(a) <= ntohs(b)) +#define WIN_GT(a,b) (ntohs(a) > ntohs(b)) +#define WIN_GEQ(a,b) (ntohs(a) >= ntohs(b)) #define WIN_MIN(a, b) ((WIN_LT(a, b)) ? (a) : (b)) #define WIN_MAX(a, b) ((WIN_GT(a, b)) ? (a) : (b)) diff --git a/freebsd/sys/netinet/tcp_subr.c b/freebsd/sys/netinet/tcp_subr.c index 1b19aecb..787213b0 100644 --- a/freebsd/sys/netinet/tcp_subr.c +++ b/freebsd/sys/netinet/tcp_subr.c @@ -40,7 +40,6 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); -#include <rtems/bsd/local/opt_compat.h> #include <rtems/bsd/local/opt_inet.h> #include <rtems/bsd/local/opt_inet6.h> #include <rtems/bsd/local/opt_ipsec.h> @@ -106,6 +105,7 @@ __FBSDID("$FreeBSD$"); #include <netinet/tcp_var.h> #include <netinet/tcp_log_buf.h> #include <netinet/tcp_syncache.h> +#include <netinet/tcp_hpts.h> #include <netinet/cc/cc.h> #ifdef INET6 #include <netinet6/tcp6_var.h> @@ -239,6 +239,9 @@ VNET_DEFINE(uma_zone_t, sack_hole_zone); VNET_DEFINE(struct hhook_head *, tcp_hhh[HHOOK_TCP_LAST+1]); #endif +static int tcp_default_fb_init(struct tcpcb *tp); +static void tcp_default_fb_fini(struct tcpcb *tp, int tcb_is_purged); +static int tcp_default_handoff_ok(struct tcpcb *tp); static struct inpcb *tcp_notify(struct inpcb *, int); static struct inpcb *tcp_mtudisc_notify(struct inpcb *, int); static void tcp_mtudisc(struct inpcb *, int); @@ -247,21 +250,17 @@ static char * tcp_log_addr(struct in_conninfo *inc, struct tcphdr *th, static struct tcp_function_block tcp_def_funcblk = { - "default", - tcp_output, - tcp_do_segment, - tcp_default_ctloutput, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - 0, - 0 + .tfb_tcp_block_name = "freebsd", + .tfb_tcp_output = tcp_output, + .tfb_tcp_do_segment = tcp_do_segment, + .tfb_tcp_ctloutput = tcp_default_ctloutput, + .tfb_tcp_handoff_ok = tcp_default_handoff_ok, + .tfb_tcp_fb_init = tcp_default_fb_init, + .tfb_tcp_fb_fini = tcp_default_fb_fini, }; int t_functions_inited = 0; +static int tcp_fb_cnt = 0; struct tcp_funchead t_functions; static struct tcp_function_block *tcp_func_set_ptr = &tcp_def_funcblk; @@ -334,6 +333,88 @@ find_and_ref_tcp_fb(struct tcp_function_block *blk) return(rblk); } +static struct tcp_function_block * +find_and_ref_tcp_default_fb(void) +{ + struct tcp_function_block *rblk; + + rw_rlock(&tcp_function_lock); + rblk = tcp_func_set_ptr; + refcount_acquire(&rblk->tfb_refcnt); + rw_runlock(&tcp_function_lock); + return (rblk); +} + +void +tcp_switch_back_to_default(struct tcpcb *tp) +{ + struct tcp_function_block *tfb; + + KASSERT(tp->t_fb != &tcp_def_funcblk, + ("%s: called by the built-in default stack", __func__)); + + /* + * Release the old stack. This function will either find a new one + * or panic. + */ + if (tp->t_fb->tfb_tcp_fb_fini != NULL) + (*tp->t_fb->tfb_tcp_fb_fini)(tp, 0); + refcount_release(&tp->t_fb->tfb_refcnt); + + /* + * Now, we'll find a new function block to use. + * Start by trying the current user-selected + * default, unless this stack is the user-selected + * default. + */ + tfb = find_and_ref_tcp_default_fb(); + if (tfb == tp->t_fb) { + refcount_release(&tfb->tfb_refcnt); + tfb = NULL; + } + /* Does the stack accept this connection? */ + if (tfb != NULL && tfb->tfb_tcp_handoff_ok != NULL && + (*tfb->tfb_tcp_handoff_ok)(tp)) { + refcount_release(&tfb->tfb_refcnt); + tfb = NULL; + } + /* Try to use that stack. */ + if (tfb != NULL) { + /* Initialize the new stack. If it succeeds, we are done. */ + tp->t_fb = tfb; + if (tp->t_fb->tfb_tcp_fb_init == NULL || + (*tp->t_fb->tfb_tcp_fb_init)(tp) == 0) + return; + + /* + * Initialization failed. Release the reference count on + * the stack. + */ + refcount_release(&tfb->tfb_refcnt); + } + + /* + * If that wasn't feasible, use the built-in default + * stack which is not allowed to reject anyone. + */ + tfb = find_and_ref_tcp_fb(&tcp_def_funcblk); + if (tfb == NULL) { + /* there always should be a default */ + panic("Can't refer to tcp_def_funcblk"); + } + if (tfb->tfb_tcp_handoff_ok != NULL) { + if ((*tfb->tfb_tcp_handoff_ok) (tp)) { + /* The default stack cannot say no */ + panic("Default stack rejects a new session?"); + } + } + tp->t_fb = tfb; + if (tp->t_fb->tfb_tcp_fb_init != NULL && + (*tp->t_fb->tfb_tcp_fb_init)(tp)) { + /* The default stack cannot fail */ + panic("Default stack initialization failed"); + } +} static int sysctl_net_inet_default_tcp_functions(SYSCTL_HANDLER_ARGS) @@ -433,14 +514,14 @@ SYSCTL_PROC(_net_inet_tcp, OID_AUTO, functions_available, "list available TCP Function sets"); /* - * Exports one (struct tcp_function_id) for each non-alias. + * Exports one (struct tcp_function_info) for each alias/name. */ static int -sysctl_net_inet_list_func_ids(SYSCTL_HANDLER_ARGS) +sysctl_net_inet_list_func_info(SYSCTL_HANDLER_ARGS) { - int error, cnt; + int cnt, error; struct tcp_function *f; - struct tcp_function_id tfi; + struct tcp_function_info tfi; /* * We don't allow writes. @@ -459,20 +540,31 @@ sysctl_net_inet_list_func_ids(SYSCTL_HANDLER_ARGS) } /* - * Walk the list, comparing the name of the function entry and - * function block to determine which is an alias. - * If exporting the list, copy out matching entries. Otherwise, - * just record the total length. + * Walk the list and copy out matching entries. If INVARIANTS + * is compiled in, also walk the list to verify the length of + * the list matches what we have recorded. */ - cnt = 0; rw_rlock(&tcp_function_lock); + + cnt = 0; +#ifndef INVARIANTS + if (req->oldptr == NULL) { + cnt = tcp_fb_cnt; + goto skip_loop; + } +#endif TAILQ_FOREACH(f, &t_functions, tf_next) { - if (strncmp(f->tf_name, f->tf_fb->tfb_tcp_block_name, - TCP_FUNCTION_NAME_LEN_MAX)) - continue; +#ifdef INVARIANTS + cnt++; +#endif if (req->oldptr != NULL) { + tfi.tfi_refcnt = f->tf_fb->tfb_refcnt; tfi.tfi_id = f->tf_fb->tfb_id; - (void)strncpy(tfi.tfi_name, f->tf_name, + (void)strncpy(tfi.tfi_alias, f->tf_name, + TCP_FUNCTION_NAME_LEN_MAX); + tfi.tfi_alias[TCP_FUNCTION_NAME_LEN_MAX - 1] = '\0'; + (void)strncpy(tfi.tfi_name, + f->tf_fb->tfb_tcp_block_name, TCP_FUNCTION_NAME_LEN_MAX); tfi.tfi_name[TCP_FUNCTION_NAME_LEN_MAX - 1] = '\0'; error = SYSCTL_OUT(req, &tfi, sizeof(tfi)); @@ -481,23 +573,110 @@ sysctl_net_inet_list_func_ids(SYSCTL_HANDLER_ARGS) * mechanism we use to accumulate length * information if the buffer was too short. */ - } else - cnt++; + } } + KASSERT(cnt == tcp_fb_cnt, + ("%s: cnt (%d) != tcp_fb_cnt (%d)", __func__, cnt, tcp_fb_cnt)); +#ifndef INVARIANTS +skip_loop: +#endif rw_runlock(&tcp_function_lock); if (req->oldptr == NULL) error = SYSCTL_OUT(req, NULL, - (cnt + 1) * sizeof(struct tcp_function_id)); + (cnt + 1) * sizeof(struct tcp_function_info)); return (error); } -SYSCTL_PROC(_net_inet_tcp, OID_AUTO, function_ids, +SYSCTL_PROC(_net_inet_tcp, OID_AUTO, function_info, CTLTYPE_OPAQUE | CTLFLAG_SKIP | CTLFLAG_RD | CTLFLAG_MPSAFE, - NULL, 0, sysctl_net_inet_list_func_ids, "S,tcp_function_id", + NULL, 0, sysctl_net_inet_list_func_info, "S,tcp_function_info", "List TCP function block name-to-ID mappings"); /* + * tfb_tcp_handoff_ok() function for the default stack. + * Note that we'll basically try to take all comers. + */ +static int +tcp_default_handoff_ok(struct tcpcb *tp) +{ + + return (0); +} + +/* + * tfb_tcp_fb_init() function for the default stack. + * + * This handles making sure we have appropriate timers set if you are + * transitioning a socket that has some amount of setup done. + * + * The init() fuction from the default can *never* return non-zero i.e. + * it is required to always succeed since it is the stack of last resort! + */ +static int +tcp_default_fb_init(struct tcpcb *tp) +{ + + struct socket *so; + + INP_WLOCK_ASSERT(tp->t_inpcb); + + KASSERT(tp->t_state >= 0 && tp->t_state < TCPS_TIME_WAIT, + ("%s: connection %p in unexpected state %d", __func__, tp, + tp->t_state)); + + /* + * Nothing to do for ESTABLISHED or LISTEN states. And, we don't + * know what to do for unexpected states (which includes TIME_WAIT). + */ + if (tp->t_state <= TCPS_LISTEN || tp->t_state >= TCPS_TIME_WAIT) + return (0); + + /* + * Make sure some kind of transmission timer is set if there is + * outstanding data. + */ + so = tp->t_inpcb->inp_socket; + if ((!TCPS_HAVEESTABLISHED(tp->t_state) || sbavail(&so->so_snd) || + tp->snd_una != tp->snd_max) && !(tcp_timer_active(tp, TT_REXMT) || + tcp_timer_active(tp, TT_PERSIST))) { + /* + * If the session has established and it looks like it should + * be in the persist state, set the persist timer. Otherwise, + * set the retransmit timer. + */ + if (TCPS_HAVEESTABLISHED(tp->t_state) && tp->snd_wnd == 0 && + (int32_t)(tp->snd_nxt - tp->snd_una) < + (int32_t)sbavail(&so->so_snd)) + tcp_setpersist(tp); + else + tcp_timer_activate(tp, TT_REXMT, tp->t_rxtcur); + } + + /* All non-embryonic sessions get a keepalive timer. */ + if (!tcp_timer_active(tp, TT_KEEP)) + tcp_timer_activate(tp, TT_KEEP, + TCPS_HAVEESTABLISHED(tp->t_state) ? TP_KEEPIDLE(tp) : + TP_KEEPINIT(tp)); + + return (0); +} + +/* + * tfb_tcp_fb_fini() function for the default stack. + * + * This changes state as necessary (or prudent) to prepare for another stack + * to assume responsibility for the connection. + */ +static void +tcp_default_fb_fini(struct tcpcb *tp, int tcb_is_purged) +{ + + INP_WLOCK_ASSERT(tp->t_inpcb); + return; +} + +/* * Target size of TCP PCB hash tables. Must be a power of two. * * Note that this can be overridden by the kernel environment @@ -660,6 +839,7 @@ register_tcp_functions_as_names(struct tcp_function_block *blk, int wait, (void)strncpy(n->tf_name, names[i], TCP_FUNCTION_NAME_LEN_MAX); n->tf_name[TCP_FUNCTION_NAME_LEN_MAX - 1] = '\0'; TAILQ_INSERT_TAIL(&t_functions, n, tf_next); + tcp_fb_cnt++; rw_wunlock(&tcp_function_lock); } return(0); @@ -676,6 +856,7 @@ cleanup: if (!strncmp(n->tf_name, names[i], TCP_FUNCTION_NAME_LEN_MAX)) { TAILQ_REMOVE(&t_functions, n, tf_next); + tcp_fb_cnt--; n->tf_fb = NULL; free(n, M_TCPFUNCTIONS); break; @@ -721,11 +902,28 @@ register_tcp_functions(struct tcp_function_block *blk, int wait) return (register_tcp_functions_as_name(blk, NULL, wait)); } +/* + * Deregister all names associated with a function block. This + * functionally removes the function block from use within the system. + * + * When called with a true quiesce argument, mark the function block + * as being removed so no more stacks will use it and determine + * whether the removal would succeed. + * + * When called with a false quiesce argument, actually attempt the + * removal. + * + * When called with a force argument, attempt to switch all TCBs to + * use the default stack instead of returning EBUSY. + * + * Returns 0 on success (or if the removal would succeed, or an error + * code on failure. + */ int -deregister_tcp_functions(struct tcp_function_block *blk) +deregister_tcp_functions(struct tcp_function_block *blk, bool quiesce, + bool force) { struct tcp_function *f; - int error=ENOENT; if (strcmp(blk->tfb_tcp_block_name, "default") == 0) { /* You can't un-register the default */ @@ -737,21 +935,64 @@ deregister_tcp_functions(struct tcp_function_block *blk) rw_wunlock(&tcp_function_lock); return (EBUSY); } + /* Mark the block so no more stacks can use it. */ + blk->tfb_flags |= TCP_FUNC_BEING_REMOVED; + /* + * If TCBs are still attached to the stack, attempt to switch them + * to the default stack. + */ + if (force && blk->tfb_refcnt) { + struct inpcb *inp; + struct tcpcb *tp; + VNET_ITERATOR_DECL(vnet_iter); + + rw_wunlock(&tcp_function_lock); + + VNET_LIST_RLOCK(); + /* XXX handle */ + VNET_FOREACH(vnet_iter) { + CURVNET_SET(vnet_iter); + INP_INFO_WLOCK(&V_tcbinfo); + LIST_FOREACH(inp, V_tcbinfo.ipi_listhead, inp_list) { + INP_WLOCK(inp); + if (inp->inp_flags & INP_TIMEWAIT) { + INP_WUNLOCK(inp); + continue; + } + tp = intotcpcb(inp); + if (tp == NULL || tp->t_fb != blk) { + INP_WUNLOCK(inp); + continue; + } + tcp_switch_back_to_default(tp); + INP_WUNLOCK(inp); + } + INP_INFO_WUNLOCK(&V_tcbinfo); + CURVNET_RESTORE(); + } + VNET_LIST_RUNLOCK(); + + rw_wlock(&tcp_function_lock); + } if (blk->tfb_refcnt) { - /* Still tcb attached, mark it. */ - blk->tfb_flags |= TCP_FUNC_BEING_REMOVED; - rw_wunlock(&tcp_function_lock); + /* TCBs still attached. */ + rw_wunlock(&tcp_function_lock); return (EBUSY); } + if (quiesce) { + /* Skip removal. */ + rw_wunlock(&tcp_function_lock); + return (0); + } + /* Remove any function names that map to this function block. */ while (find_tcp_fb_locked(blk, &f) != NULL) { - /* Found */ TAILQ_REMOVE(&t_functions, f, tf_next); + tcp_fb_cnt--; f->tf_fb = NULL; free(f, M_TCPFUNCTIONS); - error = 0; } rw_wunlock(&tcp_function_lock); - return (error); + return (0); } void @@ -1498,6 +1739,7 @@ tcp_ccalgounload(struct cc_algo *unload_algo) tmpalgo = CC_ALGO(tp); /* NewReno does not require any init. */ CC_ALGO(tp) = &newreno_cc_algo; + /* XXX defer to epoch_call */ if (tmpalgo->cb_destroy != NULL) tmpalgo->cb_destroy(tp->ccv); } @@ -1545,7 +1787,7 @@ tcp_discardcb(struct tcpcb *tp) #ifdef INET6 int isipv6 = (inp->inp_vflag & INP_IPV6) != 0; #endif /* INET6 */ - int released; + int released __unused; INP_WLOCK_ASSERT(inp); @@ -1868,6 +2110,7 @@ static int tcp_pcblist(SYSCTL_HANDLER_ARGS) { int error, i, m, n, pcb_count; + struct in_pcblist *il; struct inpcb *inp, **inp_list; inp_gen_t gencnt; struct xinpgen xig; @@ -1914,7 +2157,8 @@ tcp_pcblist(SYSCTL_HANDLER_ARGS) if (error) return (error); - inp_list = malloc(n * sizeof *inp_list, M_TEMP, M_WAITOK); + il = malloc(sizeof(struct in_pcblist) + n * sizeof(struct inpcb *), M_TEMP, M_WAITOK|M_ZERO_INVARIANTS); + inp_list = il->il_inp_list; INP_INFO_WLOCK(&V_tcbinfo); for (inp = LIST_FIRST(V_tcbinfo.ipi_listhead), i = 0; @@ -1957,14 +2201,10 @@ tcp_pcblist(SYSCTL_HANDLER_ARGS) } else INP_RUNLOCK(inp); } - INP_INFO_RLOCK(&V_tcbinfo); - for (i = 0; i < n; i++) { - inp = inp_list[i]; - INP_RLOCK(inp); - if (!in_pcbrele_rlocked(inp)) - INP_RUNLOCK(inp); - } - INP_INFO_RUNLOCK(&V_tcbinfo); + + il->il_count = n; + il->il_pcbinfo = &V_tcbinfo; + epoch_call(net_epoch_preempt, &il->il_epoch_ctx, in_pcblist_rele_rlocked); if (!error) { /* @@ -1981,7 +2221,6 @@ tcp_pcblist(SYSCTL_HANDLER_ARGS) INP_LIST_RUNLOCK(&V_tcbinfo); error = SYSCTL_OUT(req, &xig, sizeof xig); } - free(inp_list, M_TEMP); return (error); } diff --git a/freebsd/sys/netinet/tcp_syncache.c b/freebsd/sys/netinet/tcp_syncache.c index 27e0e25f..e163aa54 100644 --- a/freebsd/sys/netinet/tcp_syncache.c +++ b/freebsd/sys/netinet/tcp_syncache.c @@ -862,6 +862,12 @@ syncache_socket(struct syncache *sc, struct socket *lso, struct mbuf *m) (*tp->t_fb->tfb_tcp_fb_fini)(tp, 0); refcount_release(&tp->t_fb->tfb_refcnt); tp->t_fb = rblk; + /* + * XXXrrs this is quite dangerous, it is possible + * for the new function to fail to init. We also + * are not asking if the handoff_is_ok though at + * the very start thats probalbly ok. + */ if (tp->t_fb->tfb_tcp_fb_init) { (*tp->t_fb->tfb_tcp_fb_init)(tp); } diff --git a/freebsd/sys/netinet/tcp_timer.c b/freebsd/sys/netinet/tcp_timer.c index 69bd052b..422e5122 100644 --- a/freebsd/sys/netinet/tcp_timer.c +++ b/freebsd/sys/netinet/tcp_timer.c @@ -664,8 +664,7 @@ tcp_timer_rexmt(void * xtp) tcp_inpinfo_lock_del(inp, tp); goto out; } - tp = tcp_drop(tp, tp->t_softerror ? - tp->t_softerror : ETIMEDOUT); + tp = tcp_drop(tp, ETIMEDOUT); tcp_inpinfo_lock_del(inp, tp); goto out; } @@ -696,7 +695,12 @@ tcp_timer_rexmt(void * xtp) tp->t_flags |= TF_WASCRECOVERY; else tp->t_flags &= ~TF_WASCRECOVERY; - tp->t_badrxtwin = ticks + (tp->t_srtt >> (TCP_RTT_SHIFT + 1)); + if ((tp->t_flags & TF_RCVD_TSTMP) == 0) + tp->t_badrxtwin = ticks + (tp->t_srtt >> (TCP_RTT_SHIFT + 1)); + /* In the event that we've negotiated timestamps + * badrxtwin will be set to the value that we set + * the retransmitted packet's to_tsval to by tcp_output + */ tp->t_flags |= TF_PREVVALID; } else tp->t_flags &= ~TF_PREVVALID; diff --git a/freebsd/sys/netinet/tcp_timewait.c b/freebsd/sys/netinet/tcp_timewait.c index aa26cb37..afadf7cd 100644 --- a/freebsd/sys/netinet/tcp_timewait.c +++ b/freebsd/sys/netinet/tcp_timewait.c @@ -647,7 +647,7 @@ tcp_tw_2msl_stop(struct tcptw *tw, int reuse) { struct ucred *cred; struct inpcb *inp; - int released; + int released __unused; INP_INFO_RLOCK_ASSERT(&V_tcbinfo); diff --git a/freebsd/sys/netinet/tcp_usrreq.c b/freebsd/sys/netinet/tcp_usrreq.c index c93b2d4a..bf2cff4c 100644 --- a/freebsd/sys/netinet/tcp_usrreq.c +++ b/freebsd/sys/netinet/tcp_usrreq.c @@ -96,6 +96,7 @@ __FBSDID("$FreeBSD$"); #include <netinet/tcpip.h> #include <netinet/cc/cc.h> #include <netinet/tcp_fastopen.h> +#include <netinet/tcp_hpts.h> #ifdef TCPPCAP #include <netinet/tcp_pcap.h> #endif @@ -1097,7 +1098,9 @@ tcp_usr_abort(struct socket *so) !(inp->inp_flags & INP_DROPPED)) { tp = intotcpcb(inp); TCPDEBUG1(); - tcp_drop(tp, ECONNABORTED); + tp = tcp_drop(tp, ECONNABORTED); + if (tp == NULL) + goto dropped; TCPDEBUG2(PRU_ABORT); TCP_PROBE2(debug__user, tp, PRU_ABORT); } @@ -1108,6 +1111,7 @@ tcp_usr_abort(struct socket *so) inp->inp_flags |= INP_SOCKREF; } INP_WUNLOCK(inp); +dropped: INP_INFO_RUNLOCK(&V_tcbinfo); } @@ -1395,11 +1399,15 @@ tcp_fill_info(struct tcpcb *tp, struct tcp_info *ti) ti->tcpi_snd_nxt = tp->snd_nxt; ti->tcpi_snd_mss = tp->t_maxseg; ti->tcpi_rcv_mss = tp->t_maxseg; - if (tp->t_flags & TF_TOE) - ti->tcpi_options |= TCPI_OPT_TOE; ti->tcpi_snd_rexmitpack = tp->t_sndrexmitpack; ti->tcpi_rcv_ooopack = tp->t_rcvoopack; ti->tcpi_snd_zerowin = tp->t_sndzerowin; +#ifdef TCP_OFFLOAD + if (tp->t_flags & TF_TOE) { + ti->tcpi_options |= TCPI_OPT_TOE; + tcp_offload_tcp_info(tp, ti); + } +#endif } /* @@ -1516,22 +1524,41 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) */ (*tp->t_fb->tfb_tcp_fb_fini)(tp, 0); } +#ifdef TCPHPTS + /* Assure that we are not on any hpts */ + tcp_hpts_remove(tp->t_inpcb, HPTS_REMOVE_ALL); +#endif + if (blk->tfb_tcp_fb_init) { + error = (*blk->tfb_tcp_fb_init)(tp); + if (error) { + refcount_release(&blk->tfb_refcnt); + if (tp->t_fb->tfb_tcp_fb_init) { + if((*tp->t_fb->tfb_tcp_fb_init)(tp) != 0) { + /* Fall back failed, drop the connection */ + INP_WUNLOCK(inp); + soabort(so); + return(error); + } + } + goto err_out; + } + } refcount_release(&tp->t_fb->tfb_refcnt); tp->t_fb = blk; - if (tp->t_fb->tfb_tcp_fb_init) { - (*tp->t_fb->tfb_tcp_fb_init)(tp); - } #ifdef TCP_OFFLOAD if (tp->t_flags & TF_TOE) { tcp_offload_ctloutput(tp, sopt->sopt_dir, sopt->sopt_name); } #endif +err_out: INP_WUNLOCK(inp); return (error); } else if ((sopt->sopt_dir == SOPT_GET) && (sopt->sopt_name == TCP_FUNCTION_BLK)) { - strcpy(fsn.function_set_name, tp->t_fb->tfb_tcp_block_name); + strncpy(fsn.function_set_name, tp->t_fb->tfb_tcp_block_name, + TCP_FUNCTION_NAME_LEN_MAX); + fsn.function_set_name[TCP_FUNCTION_NAME_LEN_MAX - 1] = '\0'; fsn.pcbcnt = tp->t_fb->tfb_refcnt; INP_WUNLOCK(inp); error = sooptcopyout(sopt, &fsn, sizeof fsn); diff --git a/freebsd/sys/netinet/tcp_var.h b/freebsd/sys/netinet/tcp_var.h index f09bd19c..adaaff61 100644 --- a/freebsd/sys/netinet/tcp_var.h +++ b/freebsd/sys/netinet/tcp_var.h @@ -83,123 +83,123 @@ STAILQ_HEAD(tcp_log_stailq, tcp_log_mem); /* * Tcp control block, one per tcp; fields: - * Organized for 16 byte cacheline efficiency. + * Organized for 64 byte cacheline efficiency based + * on common tcp_input/tcp_output processing. */ struct tcpcb { - struct tsegqe_head t_segq; /* segment reassembly queue */ - int t_segqlen; /* segment reassembly queue length */ - int t_dupacks; /* consecutive dup acks recd */ - - struct tcp_timer *t_timers; /* All the TCP timers in one struct */ - + /* Cache line 1 */ struct inpcb *t_inpcb; /* back pointer to internet pcb */ - int t_state; /* state of this connection */ + struct tcp_function_block *t_fb;/* TCP function call block */ + void *t_fb_ptr; /* Pointer to t_fb specific data */ + uint32_t t_maxseg:24, /* maximum segment size */ + t_logstate:8; /* State of "black box" logging */ + uint32_t t_state:4, /* state of this connection */ + bits_spare : 24; u_int t_flags; - - struct vnet *t_vnet; /* back pointer to parent vnet */ - tcp_seq snd_una; /* sent but unacknowledged */ tcp_seq snd_max; /* highest sequence number sent; * used to recognize retransmits */ tcp_seq snd_nxt; /* send next */ tcp_seq snd_up; /* send urgent pointer */ - - tcp_seq snd_wl1; /* window update seg seq number */ - tcp_seq snd_wl2; /* window update seg ack number */ - tcp_seq iss; /* initial send sequence number */ - tcp_seq irs; /* initial receive sequence number */ - + uint32_t snd_wnd; /* send window */ + uint32_t snd_cwnd; /* congestion-controlled window */ + uint32_t cl1_spare; /* Spare to round out CL 1 */ + /* Cache line 2 */ + u_int32_t ts_offset; /* our timestamp offset */ + u_int32_t rfbuf_ts; /* recv buffer autoscaling timestamp */ + int rcv_numsacks; /* # distinct sack blks present */ + u_int t_tsomax; /* TSO total burst length limit in bytes */ + u_int t_tsomaxsegcount; /* TSO maximum segment count */ + u_int t_tsomaxsegsize; /* TSO maximum segment size in bytes */ tcp_seq rcv_nxt; /* receive next */ tcp_seq rcv_adv; /* advertised window */ uint32_t rcv_wnd; /* receive window */ + u_int t_flags2; /* More tcpcb flags storage */ + int t_srtt; /* smoothed round-trip time */ + int t_rttvar; /* variance in round-trip time */ + u_int32_t ts_recent; /* timestamp echo data */ + u_char snd_scale; /* window scaling for send window */ + u_char rcv_scale; /* window scaling for recv window */ + u_char snd_limited; /* segments limited transmitted */ + u_char request_r_scale; /* pending window scaling */ + tcp_seq last_ack_sent; + u_int t_rcvtime; /* inactivity time */ + /* Cache line 3 */ tcp_seq rcv_up; /* receive urgent pointer */ - - uint32_t snd_wnd; /* send window */ - uint32_t snd_cwnd; /* congestion-controlled window */ + int t_segqlen; /* segment reassembly queue length */ + struct tsegqe_head t_segq; /* segment reassembly queue */ + struct mbuf *t_in_pkt; + struct mbuf *t_tail_pkt; + struct tcp_timer *t_timers; /* All the TCP timers in one struct */ + struct vnet *t_vnet; /* back pointer to parent vnet */ uint32_t snd_ssthresh; /* snd_cwnd size threshold for * for slow start exponential to * linear switch */ + tcp_seq snd_wl1; /* window update seg seq number */ + /* Cache line 4 */ + tcp_seq snd_wl2; /* window update seg ack number */ + + tcp_seq irs; /* initial receive sequence number */ + tcp_seq iss; /* initial send sequence number */ + u_int t_acktime; + u_int ts_recent_age; /* when last updated */ tcp_seq snd_recover; /* for use in NewReno Fast Recovery */ + uint16_t cl4_spare; /* Spare to adjust CL 4 */ + char t_oobflags; /* have some */ + char t_iobc; /* input character */ + int t_rxtcur; /* current retransmit value (ticks) */ - u_int t_rcvtime; /* inactivity time */ - u_int t_starttime; /* time connection was established */ + int t_rxtshift; /* log(2) of rexmt exp. backoff */ u_int t_rtttime; /* RTT measurement start time */ + tcp_seq t_rtseq; /* sequence number being timed */ + u_int t_starttime; /* time connection was established */ - int t_rxtcur; /* current retransmit value (ticks) */ - u_int t_maxseg; /* maximum segment size */ u_int t_pmtud_saved_maxseg; /* pre-blackhole MSS */ - int t_srtt; /* smoothed round-trip time */ - int t_rttvar; /* variance in round-trip time */ - - int t_rxtshift; /* log(2) of rexmt exp. backoff */ u_int t_rttmin; /* minimum rtt allowed */ + u_int t_rttbest; /* best rtt we've seen */ - u_long t_rttupdated; /* number of times rtt sampled */ - uint32_t max_sndwnd; /* largest window peer has offered */ int t_softerror; /* possible error not yet reported */ -/* out-of-band data */ - char t_oobflags; /* have some */ - char t_iobc; /* input character */ -/* RFC 1323 variables */ - u_char snd_scale; /* window scaling for send window */ - u_char rcv_scale; /* window scaling for recv window */ - u_char request_r_scale; /* pending window scaling */ - u_int32_t ts_recent; /* timestamp echo data */ - u_int ts_recent_age; /* when last updated */ - u_int32_t ts_offset; /* our timestamp offset */ - - tcp_seq last_ack_sent; -/* experimental */ + uint32_t max_sndwnd; /* largest window peer has offered */ + /* Cache line 5 */ uint32_t snd_cwnd_prev; /* cwnd prior to retransmit */ uint32_t snd_ssthresh_prev; /* ssthresh prior to retransmit */ tcp_seq snd_recover_prev; /* snd_recover prior to retransmit */ int t_sndzerowin; /* zero-window updates sent */ - u_int t_badrxtwin; /* window for retransmit recovery */ - u_char snd_limited; /* segments limited transmitted */ -/* SACK related state */ + u_long t_rttupdated; /* number of times rtt sampled */ int snd_numholes; /* number of holes seen by sender */ + u_int t_badrxtwin; /* window for retransmit recovery */ TAILQ_HEAD(sackhole_head, sackhole) snd_holes; /* SACK scoreboard (sorted) */ tcp_seq snd_fack; /* last seq number(+1) sack'd by rcv'r*/ - int rcv_numsacks; /* # distinct sack blks present */ - struct sackblk sackblks[MAX_SACK_BLKS]; /* seq nos. of sack blocks */ tcp_seq sack_newdata; /* New data xmitted in this recovery episode starts at this seq number */ + struct sackblk sackblks[MAX_SACK_BLKS]; /* seq nos. of sack blocks */ struct sackhint sackhint; /* SACK scoreboard hint */ int t_rttlow; /* smallest observerved RTT */ - u_int32_t rfbuf_ts; /* recv buffer autoscaling timestamp */ int rfbuf_cnt; /* recv buffer autoscaling byte count */ struct toedev *tod; /* toedev handling this connection */ int t_sndrexmitpack; /* retransmit packets sent */ int t_rcvoopack; /* out-of-order packets received */ void *t_toe; /* TOE pcb pointer */ - int t_bytes_acked; /* # bytes acked during current RTT */ struct cc_algo *cc_algo; /* congestion control algorithm */ struct cc_var *ccv; /* congestion control specific vars */ struct osd *osd; /* storage for Khelp module data */ - + int t_bytes_acked; /* # bytes acked during current RTT */ u_int t_keepinit; /* time to establish connection */ u_int t_keepidle; /* time before keepalive probes begin */ u_int t_keepintvl; /* interval between keepalives */ u_int t_keepcnt; /* number of keepalives before close */ - - u_int t_tsomax; /* TSO total burst length limit in bytes */ - u_int t_tsomaxsegcount; /* TSO maximum segment count */ - u_int t_tsomaxsegsize; /* TSO maximum segment size in bytes */ - u_int t_flags2; /* More tcpcb flags storage */ - int t_logstate; /* State of "black box" logging */ - struct tcp_log_stailq t_logs; /* Log buffer */ + int t_dupacks; /* consecutive dup acks recd */ int t_lognum; /* Number of log entries */ - uint32_t t_logsn; /* Log "serial number" */ + struct tcp_log_stailq t_logs; /* Log buffer */ struct tcp_log_id_node *t_lin; struct tcp_log_id_bucket *t_lib; const char *t_output_caller; /* Function that called tcp_output */ - struct tcp_function_block *t_fb;/* TCP function call block */ - void *t_fb_ptr; /* Pointer to t_fb specific data */ + uint32_t t_logsn; /* Log "serial number" */ uint8_t t_tfo_client_cookie_len; /* TCP Fast Open client cookie length */ unsigned int *t_tfo_pending; /* TCP Fast Open server pending counter */ union { @@ -257,14 +257,19 @@ struct tcptemp { struct tcp_function_block { char tfb_tcp_block_name[TCP_FUNCTION_NAME_LEN_MAX]; int (*tfb_tcp_output)(struct tcpcb *); + int (*tfb_tcp_output_wtime)(struct tcpcb *, const struct timeval *); void (*tfb_tcp_do_segment)(struct mbuf *, struct tcphdr *, struct socket *, struct tcpcb *, int, int, uint8_t, int); + void (*tfb_tcp_hpts_do_segment)(struct mbuf *, struct tcphdr *, + struct socket *, struct tcpcb *, + int, int, uint8_t, + int, int, struct timeval *); int (*tfb_tcp_ctloutput)(struct socket *so, struct sockopt *sopt, struct inpcb *inp, struct tcpcb *tp); /* Optional memory allocation/free routine */ - void (*tfb_tcp_fb_init)(struct tcpcb *); + int (*tfb_tcp_fb_init)(struct tcpcb *); void (*tfb_tcp_fb_fini)(struct tcpcb *, int); /* Optional timers, must define all if you define one */ int (*tfb_tcp_timer_stop_all)(struct tcpcb *); @@ -274,6 +279,7 @@ struct tcp_function_block { void (*tfb_tcp_timer_stop)(struct tcpcb *, uint32_t); void (*tfb_tcp_rexmit_tmr)(struct tcpcb *); int (*tfb_tcp_handoff_ok)(struct tcpcb *); + void (*tfb_tcp_mtu_chg)(struct tcpcb *); volatile uint32_t tfb_refcnt; uint32_t tfb_flags; uint8_t tfb_id; @@ -688,11 +694,14 @@ void tcp_inptoxtp(const struct inpcb *, struct xtcpcb *); #endif /* - * TCP function name-to-id mapping exported to user-land via sysctl(3). + * TCP function information (name-to-id mapping, aliases, and refcnt) + * exported to user-land via sysctl(3). */ -struct tcp_function_id { +struct tcp_function_info { + uint32_t tfi_refcnt; uint8_t tfi_id; char tfi_name[TCP_FUNCTION_NAME_LEN_MAX]; + char tfi_alias[TCP_FUNCTION_NAME_LEN_MAX]; }; /* @@ -848,9 +857,12 @@ int register_tcp_functions_as_names(struct tcp_function_block *blk, int wait, const char *names[], int *num_names); int register_tcp_functions_as_name(struct tcp_function_block *blk, const char *name, int wait); -int deregister_tcp_functions(struct tcp_function_block *blk); +int deregister_tcp_functions(struct tcp_function_block *blk, bool quiesce, + bool force); struct tcp_function_block *find_and_ref_tcp_functions(struct tcp_function_set *fs); -struct tcp_function_block *find_and_ref_tcp_fb(struct tcp_function_block *blk); +void tcp_switch_back_to_default(struct tcpcb *tp); +struct tcp_function_block * +find_and_ref_tcp_fb(struct tcp_function_block *fs); int tcp_default_ctloutput(struct socket *so, struct sockopt *sopt, struct inpcb *inp, struct tcpcb *tp); uint32_t tcp_maxmtu(struct in_conninfo *, struct tcp_ifcap *); diff --git a/freebsd/sys/netinet/toecore.h b/freebsd/sys/netinet/toecore.h index 633984a6..f2374d70 100644 --- a/freebsd/sys/netinet/toecore.h +++ b/freebsd/sys/netinet/toecore.h @@ -38,6 +38,7 @@ struct tcpopt; struct tcphdr; struct in_conninfo; +struct tcp_info; struct toedev { TAILQ_ENTRY(toedev) link; /* glue for toedev_list */ @@ -101,6 +102,10 @@ struct toedev { /* TCP socket option */ void (*tod_ctloutput)(struct toedev *, struct tcpcb *, int, int); + + /* Update software state */ + void (*tod_tcp_info)(struct toedev *, struct tcpcb *, + struct tcp_info *); }; #include <sys/eventhandler.h> diff --git a/freebsd/sys/netinet/udp_usrreq.c b/freebsd/sys/netinet/udp_usrreq.c index da2dbe98..178a8d5e 100644 --- a/freebsd/sys/netinet/udp_usrreq.c +++ b/freebsd/sys/netinet/udp_usrreq.c @@ -842,6 +842,7 @@ udp_pcblist(SYSCTL_HANDLER_ARGS) { int error, i, n; struct inpcb *inp, **inp_list; + struct in_pcblist *il; inp_gen_t gencnt; struct xinpgen xig; @@ -879,10 +880,8 @@ udp_pcblist(SYSCTL_HANDLER_ARGS) error = SYSCTL_OUT(req, &xig, sizeof xig); if (error) return (error); - - inp_list = malloc(n * sizeof *inp_list, M_TEMP, M_WAITOK); - if (inp_list == NULL) - return (ENOMEM); + il = malloc(sizeof(struct in_pcblist) + n * sizeof(struct inpcb *), M_TEMP, M_WAITOK|M_ZERO_INVARIANTS); + inp_list = il->il_inp_list; INP_INFO_RLOCK(&V_udbinfo); for (inp = LIST_FIRST(V_udbinfo.ipi_listhead), i = 0; inp && i < n; @@ -911,14 +910,9 @@ udp_pcblist(SYSCTL_HANDLER_ARGS) } else INP_RUNLOCK(inp); } - INP_INFO_WLOCK(&V_udbinfo); - for (i = 0; i < n; i++) { - inp = inp_list[i]; - INP_RLOCK(inp); - if (!in_pcbrele_rlocked(inp)) - INP_RUNLOCK(inp); - } - INP_INFO_WUNLOCK(&V_udbinfo); + il->il_count = n; + il->il_pcbinfo = &V_udbinfo; + epoch_call(net_epoch_preempt, &il->il_epoch_ctx, in_pcblist_rele_rlocked); if (!error) { /* @@ -934,7 +928,6 @@ udp_pcblist(SYSCTL_HANDLER_ARGS) INP_INFO_RUNLOCK(&V_udbinfo); error = SYSCTL_OUT(req, &xig, sizeof xig); } - free(inp_list, M_TEMP); return (error); } @@ -1578,6 +1571,7 @@ udp_abort(struct socket *so) static int udp_attach(struct socket *so, int proto, struct thread *td) { + static uint32_t udp_flowid; struct inpcb *inp; struct inpcbinfo *pcbinfo; int error; @@ -1598,6 +1592,8 @@ udp_attach(struct socket *so, int proto, struct thread *td) inp = sotoinpcb(so); inp->inp_vflag |= INP_IPV4; inp->inp_ip_ttl = V_ip_defttl; + inp->inp_flowid = atomic_fetchadd_int(&udp_flowid, 1); + inp->inp_flowtype = M_HASHTYPE_OPAQUE; error = udp_newudpcb(inp); if (error) { @@ -1723,6 +1719,7 @@ udp_detach(struct socket *so) INP_WLOCK(inp); up = intoudpcb(inp); KASSERT(up != NULL, ("%s: up == NULL", __func__)); + /* XXX defer to epoch_call */ inp->inp_ppcb = NULL; in_pcbdetach(inp); in_pcbfree(inp); diff --git a/freebsd/sys/netinet6/icmp6.c b/freebsd/sys/netinet6/icmp6.c index 77876599..4d06ca16 100644 --- a/freebsd/sys/netinet6/icmp6.c +++ b/freebsd/sys/netinet6/icmp6.c @@ -385,15 +385,6 @@ icmp6_error(struct mbuf *m, int type, int code, int param) icmp6->icmp6_code = code; icmp6->icmp6_pptr = htonl((u_int32_t)param); - /* - * icmp6_reflect() is designed to be in the input path. - * icmp6_error() can be called from both input and output path, - * and if we are in output path rcvif could contain bogus value. - * clear m->m_pkthdr.rcvif for safety, we should have enough scope - * information in ip header (nip6). - */ - m->m_pkthdr.rcvif = NULL; - ICMP6STAT_INC(icp6s_outhist[type]); icmp6_reflect(m, sizeof(struct ip6_hdr)); /* header order: IPv6 - ICMPv6 */ @@ -1700,10 +1691,10 @@ ni6_addrs(struct icmp6_nodeinfo *ni6, struct mbuf *m, struct ifnet **ifpp, } IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { addrsofif = 0; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; ifa6 = (struct in6_ifaddr *)ifa; @@ -1784,12 +1775,12 @@ ni6_store_addrs(struct icmp6_nodeinfo *ni6, struct icmp6_nodeinfo *nni6, return (0); /* needless to copy */ IFNET_RLOCK_NOSLEEP(); - ifp = ifp0 ? ifp0 : TAILQ_FIRST(&V_ifnet); + ifp = ifp0 ? ifp0 : CK_STAILQ_FIRST(&V_ifnet); again: - for (; ifp; ifp = TAILQ_NEXT(ifp, if_link)) { + for (; ifp; ifp = CK_STAILQ_NEXT(ifp, if_link)) { IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; ifa6 = (struct in6_ifaddr *)ifa; @@ -2197,7 +2188,7 @@ icmp6_reflect(struct mbuf *m, size_t off) */ m->m_flags &= ~(M_BCAST|M_MCAST); - + m->m_pkthdr.rcvif = NULL; ip6_output(m, NULL, NULL, 0, NULL, &outif, NULL); if (outif) icmp6_ifoutstat_inc(outif, type, code); diff --git a/freebsd/sys/netinet6/in6.c b/freebsd/sys/netinet6/in6.c index a5d84d85..3ed80c9c 100644 --- a/freebsd/sys/netinet6/in6.c +++ b/freebsd/sys/netinet6/in6.c @@ -67,7 +67,6 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); -#include <rtems/bsd/local/opt_compat.h> #include <rtems/bsd/local/opt_inet.h> #include <rtems/bsd/local/opt_inet6.h> @@ -82,6 +81,7 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> #include <sys/priv.h> #include <sys/proc.h> +#include <sys/protosw.h> #include <sys/time.h> #include <sys/kernel.h> #include <sys/lock.h> @@ -114,6 +114,7 @@ __FBSDID("$FreeBSD$"); #include <netinet6/in6_fib.h> #include <netinet6/in6_pcb.h> + /* * struct in6_ifreq and struct ifreq must be type punnable for common members * of ifr_ifru to allow accessors to be shared. @@ -690,7 +691,6 @@ aifaddr_out: * The failure means address duplication was detected. */ } - EVENTHANDLER_INVOKE(ifaddr_event, ifp); break; } @@ -737,6 +737,30 @@ out: } +static struct in6_multi_mship * +in6_joingroup_legacy(struct ifnet *ifp, const struct in6_addr *mcaddr, + int *errorp, int delay) +{ + struct in6_multi_mship *imm; + int error; + + imm = malloc(sizeof(*imm), M_IP6MADDR, M_NOWAIT); + if (imm == NULL) { + *errorp = ENOBUFS; + return (NULL); + } + + delay = (delay * PR_FASTHZ) / hz; + + error = in6_joingroup(ifp, mcaddr, NULL, &imm->i6mm_maddr, delay); + if (error) { + *errorp = error; + free(imm, M_IP6MADDR); + return (NULL); + } + + return (imm); +} /* * Join necessary multicast groups. Factored out from in6_update_ifa(). * This entire work should only be done once, for the default FIB. @@ -773,7 +797,7 @@ in6_update_ifa_join_mc(struct ifnet *ifp, struct in6_aliasreq *ifra, */ delay = arc4random() % (MAX_RTR_SOLICITATION_DELAY * hz); } - imm = in6_joingroup(ifp, &mltaddr, &error, delay); + imm = in6_joingroup_legacy(ifp, &mltaddr, &error, delay); if (imm == NULL) { nd6log((LOG_WARNING, "%s: in6_joingroup failed for %s on %s " "(errno=%d)\n", __func__, ip6_sprintf(ip6buf, &mltaddr), @@ -790,7 +814,7 @@ in6_update_ifa_join_mc(struct ifnet *ifp, struct in6_aliasreq *ifra, if ((error = in6_setscope(&mltaddr, ifp, NULL)) != 0) goto cleanup; /* XXX: should not fail */ - imm = in6_joingroup(ifp, &mltaddr, &error, 0); + imm = in6_joingroup_legacy(ifp, &mltaddr, &error, 0); if (imm == NULL) { nd6log((LOG_WARNING, "%s: in6_joingroup failed for %s on %s " "(errno=%d)\n", __func__, ip6_sprintf(ip6buf, &mltaddr), @@ -812,7 +836,7 @@ in6_update_ifa_join_mc(struct ifnet *ifp, struct in6_aliasreq *ifra, } if (in6_nigroup(ifp, NULL, -1, &mltaddr) == 0) { /* XXX jinmei */ - imm = in6_joingroup(ifp, &mltaddr, &error, delay); + imm = in6_joingroup_legacy(ifp, &mltaddr, &error, delay); if (imm == NULL) nd6log((LOG_WARNING, "%s: in6_joingroup failed for %s on %s " @@ -824,7 +848,7 @@ in6_update_ifa_join_mc(struct ifnet *ifp, struct in6_aliasreq *ifra, } if (V_icmp6_nodeinfo_oldmcprefix && in6_nigroup_oldmcprefix(ifp, NULL, -1, &mltaddr) == 0) { - imm = in6_joingroup(ifp, &mltaddr, &error, delay); + imm = in6_joingroup_legacy(ifp, &mltaddr, &error, delay); if (imm == NULL) nd6log((LOG_WARNING, "%s: in6_joingroup failed for %s on %s " @@ -843,7 +867,7 @@ in6_update_ifa_join_mc(struct ifnet *ifp, struct in6_aliasreq *ifra, if ((error = in6_setscope(&mltaddr, ifp, NULL)) != 0) goto cleanup; /* XXX: should not fail */ - imm = in6_joingroup(ifp, &mltaddr, &error, 0); + imm = in6_joingroup_legacy(ifp, &mltaddr, &error, 0); if (imm == NULL) { nd6log((LOG_WARNING, "%s: in6_joingroup failed for %s on %s " "(errno=%d)\n", __func__, ip6_sprintf(ip6buf, @@ -1099,13 +1123,13 @@ in6_alloc_ifa(struct ifnet *ifp, struct in6_aliasreq *ifra, int flags) ia->ia_ifp = ifp; ifa_ref(&ia->ia_ifa); /* if_addrhead */ IF_ADDR_WLOCK(ifp); - TAILQ_INSERT_TAIL(&ifp->if_addrhead, &ia->ia_ifa, ifa_link); + CK_STAILQ_INSERT_TAIL(&ifp->if_addrhead, &ia->ia_ifa, ifa_link); IF_ADDR_WUNLOCK(ifp); ifa_ref(&ia->ia_ifa); /* in6_ifaddrhead */ IN6_IFADDR_WLOCK(); - TAILQ_INSERT_TAIL(&V_in6_ifaddrhead, ia, ia_link); - LIST_INSERT_HEAD(IN6ADDR_HASH(&ia->ia_addr.sin6_addr), ia, ia6_hash); + CK_STAILQ_INSERT_TAIL(&V_in6_ifaddrhead, ia, ia_link); + CK_LIST_INSERT_HEAD(IN6ADDR_HASH(&ia->ia_addr.sin6_addr), ia, ia6_hash); IN6_IFADDR_WUNLOCK(); return (ia); @@ -1278,7 +1302,9 @@ in6_purgeaddr(struct ifaddr *ifa) /* Leave multicast groups. */ while ((imm = LIST_FIRST(&ia->ia6_memberships)) != NULL) { LIST_REMOVE(imm, i6mm_chain); - in6_leavegroup(imm); + if (imm->i6mm_maddr != NULL) + in6_leavegroup(imm->i6mm_maddr, NULL); + free(imm, M_IP6MADDR); } plen = in6_mask2len(&ia->ia_prefixmask.sin6_addr, NULL); /* XXX */ if ((ia->ia_flags & IFA_ROUTE) && plen == 128) { @@ -1301,7 +1327,7 @@ in6_unlink_ifa(struct in6_ifaddr *ia, struct ifnet *ifp) int remove_lle; IF_ADDR_WLOCK(ifp); - TAILQ_REMOVE(&ifp->if_addrhead, &ia->ia_ifa, ifa_link); + CK_STAILQ_REMOVE(&ifp->if_addrhead, &ia->ia_ifa, ifaddr, ifa_link); IF_ADDR_WUNLOCK(ifp); ifa_free(&ia->ia_ifa); /* if_addrhead */ @@ -1311,8 +1337,8 @@ in6_unlink_ifa(struct in6_ifaddr *ia, struct ifnet *ifp) * cleanup. */ IN6_IFADDR_WLOCK(); - TAILQ_REMOVE(&V_in6_ifaddrhead, ia, ia_link); - LIST_REMOVE(ia, ia6_hash); + CK_STAILQ_REMOVE(&V_in6_ifaddrhead, ia, in6_ifaddr, ia_link); + CK_LIST_REMOVE(ia, ia6_hash); IN6_IFADDR_WUNLOCK(); /* @@ -1366,7 +1392,7 @@ in6_notify_ifa(struct ifnet *ifp, struct in6_ifaddr *ia, */ if (hostIsNew != 0) { IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; ifacount++; @@ -1377,7 +1403,7 @@ in6_notify_ifa(struct ifnet *ifp, struct in6_ifaddr *ia, if (ifacount <= 1 && ifp->if_ioctl) { error = (*ifp->if_ioctl)(ifp, SIOCSIFADDR, (caddr_t)ia); if (error) - return (error); + goto done; } /* @@ -1417,7 +1443,7 @@ in6_notify_ifa(struct ifnet *ifp, struct in6_ifaddr *ia, ia->ia_flags |= IFA_RTSELF; error = rtinit(&ia->ia_ifa, RTM_ADD, ia->ia_flags | rtflags); if (error) - return (error); + goto done; ia->ia_flags |= IFA_ROUTE; } @@ -1430,6 +1456,11 @@ in6_notify_ifa(struct ifnet *ifp, struct in6_ifaddr *ia, if (error == 0) ia->ia_flags |= IFA_RTSELF; } +done: + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, + "Invoking IPv6 network device address event may sleep"); + + EVENTHANDLER_INVOKE(ifaddr_event, ifp); return (error); } @@ -1444,7 +1475,7 @@ in6ifa_ifpforlinklocal(struct ifnet *ifp, int ignoreflags) struct ifaddr *ifa; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; if (IN6_IS_ADDR_LINKLOCAL(IFA_IN6(ifa))) { @@ -1472,7 +1503,7 @@ in6ifa_ifwithaddr(const struct in6_addr *addr, uint32_t zoneid) struct in6_ifaddr *ia; IN6_IFADDR_RLOCK(&in6_ifa_tracker); - LIST_FOREACH(ia, IN6ADDR_HASH(addr), ia6_hash) { + CK_LIST_FOREACH(ia, IN6ADDR_HASH(addr), ia6_hash) { if (IN6_ARE_ADDR_EQUAL(IA6_IN6(ia), addr)) { if (zoneid != 0 && zoneid != ia->ia_addr.sin6_scope_id) @@ -1495,7 +1526,7 @@ in6ifa_ifpwithaddr(struct ifnet *ifp, const struct in6_addr *addr) struct ifaddr *ifa; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; if (IN6_ARE_ADDR_EQUAL(addr, IFA_IN6(ifa))) { @@ -1520,7 +1551,7 @@ in6ifa_llaonifp(struct ifnet *ifp) if (ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) return (NULL); IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; sin6 = (struct sockaddr_in6 *)ifa->ifa_addr; @@ -1626,7 +1657,7 @@ in6_localaddr(struct in6_addr *in6) return 1; IN6_IFADDR_RLOCK(&in6_ifa_tracker); - TAILQ_FOREACH(ia, &V_in6_ifaddrhead, ia_link) { + CK_STAILQ_FOREACH(ia, &V_in6_ifaddrhead, ia_link) { if (IN6_ARE_MASKED_ADDR_EQUAL(in6, &ia->ia_addr.sin6_addr, &ia->ia_prefixmask.sin6_addr)) { IN6_IFADDR_RUNLOCK(&in6_ifa_tracker); @@ -1649,7 +1680,7 @@ in6_localip(struct in6_addr *in6) struct in6_ifaddr *ia; IN6_IFADDR_RLOCK(&in6_ifa_tracker); - LIST_FOREACH(ia, IN6ADDR_HASH(in6), ia6_hash) { + CK_LIST_FOREACH(ia, IN6ADDR_HASH(in6), ia6_hash) { if (IN6_ARE_ADDR_EQUAL(in6, &ia->ia_addr.sin6_addr)) { IN6_IFADDR_RUNLOCK(&in6_ifa_tracker); return (1); @@ -1675,7 +1706,7 @@ in6_ifhasaddr(struct ifnet *ifp, struct in6_addr *addr) in6_setscope(&in6, ifp, NULL); IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; ia6 = (struct in6_ifaddr *)ifa; @@ -1696,7 +1727,7 @@ in6_is_addr_deprecated(struct sockaddr_in6 *sa6) struct in6_ifaddr *ia; IN6_IFADDR_RLOCK(&in6_ifa_tracker); - LIST_FOREACH(ia, IN6ADDR_HASH(&sa6->sin6_addr), ia6_hash) { + CK_LIST_FOREACH(ia, IN6ADDR_HASH(&sa6->sin6_addr), ia6_hash) { if (IN6_ARE_ADDR_EQUAL(IA6_IN6(ia), &sa6->sin6_addr)) { if (ia->ia6_flags & IN6_IFF_DEPRECATED) { IN6_IFADDR_RUNLOCK(&in6_ifa_tracker); @@ -1802,7 +1833,7 @@ in6_ifawithifp(struct ifnet *ifp, struct in6_addr *dst) * If none, return one of global addresses assigned other ifs. */ IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; if (((struct in6_ifaddr *)ifa)->ia6_flags & IN6_IFF_ANYCAST) @@ -1839,7 +1870,7 @@ in6_ifawithifp(struct ifnet *ifp, struct in6_addr *dst) return (besta); } - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; if (((struct in6_ifaddr *)ifa)->ia6_flags & IN6_IFF_ANYCAST) @@ -1886,7 +1917,7 @@ in6_if_up(struct ifnet *ifp) struct in6_ifaddr *ia; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; ia = (struct in6_ifaddr *)ifa; @@ -1946,7 +1977,7 @@ in6_setmaxmtu(void) struct ifnet *ifp; IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { /* this function can be called during ifnet initialization */ if (!ifp->if_afdata[AF_INET6]) continue; @@ -1977,12 +2008,8 @@ in6_if2idlen(struct ifnet *ifp) case IFT_BRIDGE: /* bridge(4) only does Ethernet-like links */ case IFT_INFINIBAND: return (64); - case IFT_FDDI: /* RFC2467 */ - return (64); case IFT_PPP: /* RFC2472 */ return (64); - case IFT_ARCNET: /* RFC2497 */ - return (64); case IFT_FRELAY: /* RFC2590 */ return (64); case IFT_IEEE1394: /* RFC3146 */ @@ -2022,9 +2049,11 @@ struct in6_llentry { * Do actual deallocation of @lle. */ static void -in6_lltable_destroy_lle_unlocked(struct llentry *lle) +in6_lltable_destroy_lle_unlocked(epoch_context_t ctx) { + struct llentry *lle; + lle = __containerof(ctx, struct llentry, lle_epoch_ctx); LLE_LOCK_DESTROY(lle); LLE_REQ_DESTROY(lle); free(lle, M_LLTABLE); @@ -2039,7 +2068,7 @@ in6_lltable_destroy_lle(struct llentry *lle) { LLE_WUNLOCK(lle); - in6_lltable_destroy_lle_unlocked(lle); + epoch_call(net_epoch_preempt, &lle->lle_epoch_ctx, in6_lltable_destroy_lle_unlocked); } static struct llentry * @@ -2142,11 +2171,13 @@ in6_lltable_rtcheck(struct ifnet *ifp, * Create an ND6 cache for an IPv6 neighbor * that is not covered by our own prefix. */ + NET_EPOCH_ENTER(); ifa = ifaof_ifpforaddr(l3addr, ifp); if (ifa != NULL) { - ifa_free(ifa); + NET_EPOCH_EXIT(); return 0; } + NET_EPOCH_EXIT(); log(LOG_INFO, "IPv6 address: \"%s\" is not on the network\n", ip6_sprintf(ip6buf, &sin6->sin6_addr)); return EINVAL; @@ -2208,7 +2239,7 @@ in6_lltable_find_dst(struct lltable *llt, const struct in6_addr *dst) hashidx = in6_lltable_hash_dst(dst, llt->llt_hsize); lleh = &llt->lle_head[hashidx]; - LIST_FOREACH(lle, lleh, lle_next) { + CK_LIST_FOREACH(lle, lleh, lle_next) { if (lle->la_flags & LLE_DELETED) continue; if (IN6_ARE_ADDR_EQUAL(&lle->r_l3addr.addr6, dst)) @@ -2263,7 +2294,7 @@ in6_lltable_alloc(struct lltable *llt, u_int flags, linkhdrsize = LLE_MAX_LINKHDR; if (lltable_calc_llheader(ifp, AF_INET6, IF_LLADDR(ifp), linkhdr, &linkhdrsize, &lladdr_off) != 0) { - in6_lltable_destroy_lle_unlocked(lle); + epoch_call(net_epoch_preempt, &lle->lle_epoch_ctx, in6_lltable_destroy_lle_unlocked); return (NULL); } lltable_set_entry_addr(ifp, lle, linkhdr, linkhdrsize, @@ -2328,62 +2359,58 @@ in6_lltable_dump_entry(struct lltable *llt, struct llentry *lle, int error; bzero(&ndpc, sizeof(ndpc)); - /* skip deleted entries */ - if ((lle->la_flags & LLE_DELETED) == LLE_DELETED) - return (0); - /* Skip if jailed and not a valid IP of the prison. */ - lltable_fill_sa_entry(lle, - (struct sockaddr *)&ndpc.sin6); - if (prison_if(wr->td->td_ucred, - (struct sockaddr *)&ndpc.sin6) != 0) - return (0); - /* - * produce a msg made of: - * struct rt_msghdr; - * struct sockaddr_in6 (IPv6) - * struct sockaddr_dl; - */ - ndpc.rtm.rtm_msglen = sizeof(ndpc); - ndpc.rtm.rtm_version = RTM_VERSION; - ndpc.rtm.rtm_type = RTM_GET; - ndpc.rtm.rtm_flags = RTF_UP; - ndpc.rtm.rtm_addrs = RTA_DST | RTA_GATEWAY; - if (V_deembed_scopeid) - sa6_recoverscope(&ndpc.sin6); - - /* publish */ - if (lle->la_flags & LLE_PUB) - ndpc.rtm.rtm_flags |= RTF_ANNOUNCE; - - sdl = &ndpc.sdl; - sdl->sdl_family = AF_LINK; - sdl->sdl_len = sizeof(*sdl); - sdl->sdl_index = ifp->if_index; - sdl->sdl_type = ifp->if_type; - if ((lle->la_flags & LLE_VALID) == LLE_VALID) { - sdl->sdl_alen = ifp->if_addrlen; - bcopy(lle->ll_addr, LLADDR(sdl), - ifp->if_addrlen); - } else { - sdl->sdl_alen = 0; - bzero(LLADDR(sdl), ifp->if_addrlen); - } - if (lle->la_expire != 0) - ndpc.rtm.rtm_rmx.rmx_expire = lle->la_expire + - lle->lle_remtime / hz + - time_second - time_uptime; - ndpc.rtm.rtm_flags |= (RTF_HOST | RTF_LLDATA); - if (lle->la_flags & LLE_STATIC) - ndpc.rtm.rtm_flags |= RTF_STATIC; - if (lle->la_flags & LLE_IFADDR) - ndpc.rtm.rtm_flags |= RTF_PINNED; - if (lle->ln_router != 0) - ndpc.rtm.rtm_flags |= RTF_GATEWAY; - ndpc.rtm.rtm_rmx.rmx_pksent = lle->la_asked; - /* Store state in rmx_weight value */ - ndpc.rtm.rtm_rmx.rmx_state = lle->ln_state; - ndpc.rtm.rtm_index = ifp->if_index; - error = SYSCTL_OUT(wr, &ndpc, sizeof(ndpc)); + /* skip deleted entries */ + if ((lle->la_flags & LLE_DELETED) == LLE_DELETED) + return (0); + /* Skip if jailed and not a valid IP of the prison. */ + lltable_fill_sa_entry(lle, (struct sockaddr *)&ndpc.sin6); + if (prison_if(wr->td->td_ucred, (struct sockaddr *)&ndpc.sin6) != 0) + return (0); + /* + * produce a msg made of: + * struct rt_msghdr; + * struct sockaddr_in6 (IPv6) + * struct sockaddr_dl; + */ + ndpc.rtm.rtm_msglen = sizeof(ndpc); + ndpc.rtm.rtm_version = RTM_VERSION; + ndpc.rtm.rtm_type = RTM_GET; + ndpc.rtm.rtm_flags = RTF_UP; + ndpc.rtm.rtm_addrs = RTA_DST | RTA_GATEWAY; + if (V_deembed_scopeid) + sa6_recoverscope(&ndpc.sin6); + + /* publish */ + if (lle->la_flags & LLE_PUB) + ndpc.rtm.rtm_flags |= RTF_ANNOUNCE; + + sdl = &ndpc.sdl; + sdl->sdl_family = AF_LINK; + sdl->sdl_len = sizeof(*sdl); + sdl->sdl_index = ifp->if_index; + sdl->sdl_type = ifp->if_type; + if ((lle->la_flags & LLE_VALID) == LLE_VALID) { + sdl->sdl_alen = ifp->if_addrlen; + bcopy(lle->ll_addr, LLADDR(sdl), ifp->if_addrlen); + } else { + sdl->sdl_alen = 0; + bzero(LLADDR(sdl), ifp->if_addrlen); + } + if (lle->la_expire != 0) + ndpc.rtm.rtm_rmx.rmx_expire = lle->la_expire + + lle->lle_remtime / hz + time_second - time_uptime; + ndpc.rtm.rtm_flags |= (RTF_HOST | RTF_LLDATA); + if (lle->la_flags & LLE_STATIC) + ndpc.rtm.rtm_flags |= RTF_STATIC; + if (lle->la_flags & LLE_IFADDR) + ndpc.rtm.rtm_flags |= RTF_PINNED; + if (lle->ln_router != 0) + ndpc.rtm.rtm_flags |= RTF_GATEWAY; + ndpc.rtm.rtm_rmx.rmx_pksent = lle->la_asked; + /* Store state in rmx_weight value */ + ndpc.rtm.rtm_rmx.rmx_state = lle->ln_state; + ndpc.rtm.rtm_index = ifp->if_index; + error = SYSCTL_OUT(wr, &ndpc, sizeof(ndpc)); return (error); } diff --git a/freebsd/sys/netinet6/in6_ifattach.c b/freebsd/sys/netinet6/in6_ifattach.c index 1ce489f5..81182b4e 100644 --- a/freebsd/sys/netinet6/in6_ifattach.c +++ b/freebsd/sys/netinet6/in6_ifattach.c @@ -255,7 +255,7 @@ in6_get_hw_ifid(struct ifnet *ifp, struct in6_addr *in6) { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_LINK) continue; sdl = (struct sockaddr_dl *)ifa->ifa_addr; @@ -280,7 +280,6 @@ found: case IFT_BRIDGE: case IFT_ETHER: case IFT_L2VLAN: - case IFT_FDDI: case IFT_ATM: case IFT_IEEE1394: /* IEEE802/EUI64 cases - what others? */ @@ -323,26 +322,6 @@ found: } break; - case IFT_ARCNET: - if (addrlen != 1) { - IF_ADDR_RUNLOCK(ifp); - return -1; - } - if (!addr[0]) { - IF_ADDR_RUNLOCK(ifp); - return -1; - } - - bzero(&in6->s6_addr[8], 8); - in6->s6_addr[15] = addr[0]; - - /* - * due to insufficient bitwidth, we mark it local. - */ - in6->s6_addr[8] &= ~EUI64_GBIT; /* g bit to "individual" */ - in6->s6_addr[8] |= EUI64_UBIT; /* u bit to "local" */ - break; - case IFT_GIF: case IFT_STF: /* @@ -411,7 +390,7 @@ get_ifid(struct ifnet *ifp0, struct ifnet *altifp, /* next, try to get it from some other hardware interface */ IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (ifp == ifp0) continue; if (in6_get_hw_ifid(ifp, in6) != 0) @@ -782,7 +761,7 @@ _in6_ifdetach(struct ifnet *ifp, int purgeulp) * nuke any of IPv6 addresses we have * XXX: all addresses should be already removed */ - TAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, next) { + CK_STAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, next) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; in6_purgeaddr(ifa); @@ -857,7 +836,7 @@ in6_tmpaddrtimer(void *arg) V_ip6_temp_regen_advance) * hz, in6_tmpaddrtimer, curvnet); bzero(nullbuf, sizeof(nullbuf)); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (ifp->if_afdata[AF_INET6] == NULL) continue; ndi = ND_IFINFO(ifp); @@ -877,36 +856,35 @@ in6_tmpaddrtimer(void *arg) static void in6_purgemaddrs(struct ifnet *ifp) { - LIST_HEAD(,in6_multi) purgeinms; - struct in6_multi *inm, *tinm; - struct ifmultiaddr *ifma; + struct in6_multi_head purgeinms; + struct in6_multi *inm; + struct ifmultiaddr *ifma, *next; - LIST_INIT(&purgeinms); + SLIST_INIT(&purgeinms); IN6_MULTI_LOCK(); - + IN6_MULTI_LIST_LOCK(); + IF_ADDR_WLOCK(ifp); /* * Extract list of in6_multi associated with the detaching ifp * which the PF_INET6 layer is about to release. - * We need to do this as IF_ADDR_LOCK() may be re-acquired - * by code further down. */ - IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + restart: + CK_STAILQ_FOREACH_SAFE(ifma, &ifp->if_multiaddrs, ifma_link, next) { if (ifma->ifma_addr->sa_family != AF_INET6 || ifma->ifma_protospec == NULL) continue; inm = (struct in6_multi *)ifma->ifma_protospec; - LIST_INSERT_HEAD(&purgeinms, inm, in6m_entry); - } - IF_ADDR_RUNLOCK(ifp); - - LIST_FOREACH_SAFE(inm, &purgeinms, in6m_entry, tinm) { - LIST_REMOVE(inm, in6m_entry); - in6m_release_locked(inm); + in6m_rele_locked(&purgeinms, inm); + if (__predict_false(ifma6_restart)) { + ifma6_restart = false; + goto restart; + } } + IF_ADDR_WUNLOCK(ifp); mld_ifdetach(ifp); - + IN6_MULTI_LIST_UNLOCK(); IN6_MULTI_UNLOCK(); + in6m_release_list_deferred(&purgeinms); } void diff --git a/freebsd/sys/netinet6/in6_mcast.c b/freebsd/sys/netinet6/in6_mcast.c index a634b18b..32660c89 100644 --- a/freebsd/sys/netinet6/in6_mcast.c +++ b/freebsd/sys/netinet6/in6_mcast.c @@ -43,13 +43,13 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/systm.h> +#include <sys/gtaskqueue.h> #include <sys/kernel.h> #include <sys/malloc.h> #include <sys/mbuf.h> #include <sys/protosw.h> #include <sys/socket.h> #include <sys/socketvar.h> -#include <sys/protosw.h> #include <sys/sysctl.h> #include <sys/priv.h> #include <sys/ktr.h> @@ -61,8 +61,12 @@ __FBSDID("$FreeBSD$"); #include <net/route.h> #include <net/vnet.h> + #include <netinet/in.h> +#include <netinet/udp.h> #include <netinet/in_var.h> +#include <netinet/ip_var.h> +#include <netinet/udp_var.h> #include <netinet6/in6_fib.h> #include <netinet6/in6_var.h> #include <netinet/ip6.h> @@ -91,7 +95,7 @@ typedef union sockunion sockunion_t; static MALLOC_DEFINE(M_IN6MFILTER, "in6_mfilter", "IPv6 multicast PCB-layer source filter"); -static MALLOC_DEFINE(M_IP6MADDR, "in6_multi", "IPv6 multicast group"); +MALLOC_DEFINE(M_IP6MADDR, "in6_multi", "IPv6 multicast group"); static MALLOC_DEFINE(M_IP6MOPTS, "ip6_moptions", "IPv6 multicast options"); static MALLOC_DEFINE(M_IP6MSOURCE, "ip6_msource", "IPv6 multicast MLD-layer source filter"); @@ -109,8 +113,16 @@ RB_GENERATE(ip6_msource_tree, ip6_msource, im6s_link, ip6_msource_cmp); * any need for in6_multi itself to be virtualized -- it is bound to an ifp * anyway no matter what happens. */ -struct mtx in6_multi_mtx; -MTX_SYSINIT(in6_multi_mtx, &in6_multi_mtx, "in6_multi_mtx", MTX_DEF); +struct mtx in6_multi_list_mtx; +MTX_SYSINIT(in6_multi_mtx, &in6_multi_list_mtx, "in6_multi_list_mtx", MTX_DEF); + +struct mtx in6_multi_free_mtx; +MTX_SYSINIT(in6_multi_free_mtx, &in6_multi_free_mtx, "in6_multi_free_mtx", MTX_DEF); + +struct sx in6_multi_sx; +SX_SYSINIT(in6_multi_sx, &in6_multi_sx, "in6_multi_sx"); + + static void im6f_commit(struct in6_mfilter *); static int im6f_get_source(struct in6_mfilter *imf, @@ -132,7 +144,7 @@ static struct in6_msource * const struct sockaddr *); static void im6s_merge(struct ip6_msource *ims, const struct in6_msource *lims, const int rollback); -static int in6_mc_get(struct ifnet *, const struct in6_addr *, +static int in6_getmulti(struct ifnet *, const struct in6_addr *, struct in6_multi **); static int in6m_get_source(struct in6_multi *inm, const struct in6_addr *addr, const int noalloc, @@ -180,6 +192,7 @@ static SYSCTL_NODE(_net_inet6_ip6_mcast, OID_AUTO, filters, CTLFLAG_RD | CTLFLAG_MPSAFE, sysctl_ip6_mcast_filters, "Per-interface stack-wide source filters"); +int ifma6_restart = 0; #ifdef KTR /* * Inline function which wraps assertions for a valid ifp. @@ -391,7 +404,7 @@ im6o_mc_filter(const struct ip6_moptions *imo, const struct ifnet *ifp, * Return 0 if successful, otherwise return an appropriate error code. */ static int -in6_mc_get(struct ifnet *ifp, const struct in6_addr *group, +in6_getmulti(struct ifnet *ifp, const struct in6_addr *group, struct in6_multi **pinm) { struct sockaddr_in6 gsin6; @@ -407,8 +420,8 @@ in6_mc_get(struct ifnet *ifp, const struct in6_addr *group, * re-acquire around the call. */ IN6_MULTI_LOCK_ASSERT(); + IN6_MULTI_LIST_LOCK(); IF_ADDR_WLOCK(ifp); - inm = in6m_lookup_locked(ifp, group); if (inm != NULL) { /* @@ -417,7 +430,7 @@ in6_mc_get(struct ifnet *ifp, const struct in6_addr *group, */ KASSERT(inm->in6m_refcount >= 1, ("%s: bad refcount %d", __func__, inm->in6m_refcount)); - ++inm->in6m_refcount; + in6m_acquire_locked(inm); *pinm = inm; goto out_locked; } @@ -431,10 +444,12 @@ in6_mc_get(struct ifnet *ifp, const struct in6_addr *group, * Check if a link-layer group is already associated * with this network-layer group on the given ifnet. */ + IN6_MULTI_LIST_UNLOCK(); IF_ADDR_WUNLOCK(ifp); error = if_addmulti(ifp, (struct sockaddr *)&gsin6, &ifma); if (error != 0) return (error); + IN6_MULTI_LIST_LOCK(); IF_ADDR_WLOCK(ifp); /* @@ -457,7 +472,7 @@ in6_mc_get(struct ifnet *ifp, const struct in6_addr *group, panic("%s: ifma %p is inconsistent with %p (%p)", __func__, ifma, inm, group); #endif - ++inm->in6m_refcount; + in6m_acquire_locked(inm); *pinm = inm; goto out_locked; } @@ -474,6 +489,7 @@ in6_mc_get(struct ifnet *ifp, const struct in6_addr *group, */ inm = malloc(sizeof(*inm), M_IP6MADDR, M_NOWAIT | M_ZERO); if (inm == NULL) { + IN6_MULTI_LIST_UNLOCK(); IF_ADDR_WUNLOCK(ifp); if_delmulti_ifma(ifma); return (ENOMEM); @@ -493,7 +509,8 @@ in6_mc_get(struct ifnet *ifp, const struct in6_addr *group, ifma->ifma_protospec = inm; *pinm = inm; -out_locked: + out_locked: + IN6_MULTI_LIST_UNLOCK(); IF_ADDR_WUNLOCK(ifp); return (error); } @@ -504,36 +521,139 @@ out_locked: * If the refcount drops to 0, free the in6_multi record and * delete the underlying link-layer membership. */ -void -in6m_release_locked(struct in6_multi *inm) +static void +in6m_release(struct in6_multi *inm) { struct ifmultiaddr *ifma; - - IN6_MULTI_LOCK_ASSERT(); + struct ifnet *ifp; CTR2(KTR_MLD, "%s: refcount is %d", __func__, inm->in6m_refcount); - if (--inm->in6m_refcount > 0) { - CTR2(KTR_MLD, "%s: refcount is now %d", __func__, - inm->in6m_refcount); - return; - } - + MPASS(inm->in6m_refcount == 0); CTR2(KTR_MLD, "%s: freeing inm %p", __func__, inm); ifma = inm->in6m_ifma; + ifp = inm->in6m_ifp; + MPASS(ifma->ifma_llifma == NULL); /* XXX this access is not covered by IF_ADDR_LOCK */ CTR2(KTR_MLD, "%s: purging ifma %p", __func__, ifma); - KASSERT(ifma->ifma_protospec == inm, - ("%s: ifma_protospec != inm", __func__)); - ifma->ifma_protospec = NULL; + KASSERT(ifma->ifma_protospec == NULL, + ("%s: ifma_protospec != NULL", __func__)); - in6m_purge(inm); + if (ifp != NULL) { + CURVNET_SET(ifp->if_vnet); + in6m_purge(inm); + free(inm, M_IP6MADDR); + if_delmulti_ifma_flags(ifma, 1); + CURVNET_RESTORE(); + if_rele(ifp); + } else { + in6m_purge(inm); + free(inm, M_IP6MADDR); + if_delmulti_ifma_flags(ifma, 1); + } +} + +static struct grouptask free_gtask; +static struct in6_multi_head in6m_free_list; +static void in6m_release_task(void *arg __unused); +static void in6m_init(void) +{ + SLIST_INIT(&in6m_free_list); + taskqgroup_config_gtask_init(NULL, &free_gtask, in6m_release_task, "in6m release task"); +} - free(inm, M_IP6MADDR); +SYSINIT(in6m_init, SI_SUB_SMP + 1, SI_ORDER_FIRST, + in6m_init, NULL); - if_delmulti_ifma(ifma); + +void +in6m_release_list_deferred(struct in6_multi_head *inmh) +{ + if (SLIST_EMPTY(inmh)) + return; + mtx_lock(&in6_multi_free_mtx); + SLIST_CONCAT(&in6m_free_list, inmh, in6_multi, in6m_nrele); + mtx_unlock(&in6_multi_free_mtx); + GROUPTASK_ENQUEUE(&free_gtask); +} + +void +in6m_disconnect(struct in6_multi *inm) +{ + struct ifnet *ifp; + struct ifaddr *ifa; + struct in6_ifaddr *ifa6; + struct in6_multi_mship *imm, *imm_tmp; + struct ifmultiaddr *ifma, *ll_ifma; + + ifp = inm->in6m_ifp; + IF_ADDR_WLOCK_ASSERT(ifp); + ifma = inm->in6m_ifma; + + if_ref(ifp); + CK_STAILQ_REMOVE(&ifp->if_multiaddrs, ifma, ifmultiaddr, ifma_link); + MCDPRINTF("removed ifma: %p from %s\n", ifma, ifp->if_xname); + if ((ll_ifma = ifma->ifma_llifma) != NULL) { + MPASS(ifma != ll_ifma); + ifma->ifma_llifma = NULL; + MPASS(ll_ifma->ifma_llifma == NULL); + MPASS(ll_ifma->ifma_ifp == ifp); + if (--ll_ifma->ifma_refcount == 0) { + ifma6_restart = true; + CK_STAILQ_REMOVE(&ifp->if_multiaddrs, ll_ifma, ifmultiaddr, ifma_link); + MCDPRINTF("removed ll_ifma: %p from %s\n", ll_ifma, ifp->if_xname); + if_freemulti(ll_ifma); + } + } + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + if (ifa->ifa_addr->sa_family != AF_INET6) + continue; + ifa6 = (void *)ifa; + LIST_FOREACH_SAFE(imm, &ifa6->ia6_memberships, + i6mm_chain, imm_tmp) { + if (inm == imm->i6mm_maddr) { + LIST_REMOVE(imm, i6mm_chain); + free(imm, M_IP6MADDR); + } + } + } +} + +void +in6m_release_deferred(struct in6_multi *inm) +{ + struct in6_multi_head tmp; + + IN6_MULTI_LIST_LOCK_ASSERT(); + KASSERT(inm->in6m_refcount > 0, ("refcount == %d inm: %p", inm->in6m_refcount, inm)); + if (--inm->in6m_refcount == 0) { + in6m_disconnect(inm); + SLIST_INIT(&tmp); + inm->in6m_ifma->ifma_protospec = NULL; + MPASS(inm->in6m_ifma->ifma_llifma == NULL); + SLIST_INSERT_HEAD(&tmp, inm, in6m_nrele); + in6m_release_list_deferred(&tmp); + } +} + +static void +in6m_release_task(void *arg __unused) +{ + struct in6_multi_head in6m_free_tmp; + struct in6_multi *inm, *tinm; + + SLIST_INIT(&in6m_free_tmp); + mtx_lock(&in6_multi_free_mtx); + SLIST_CONCAT(&in6m_free_tmp, &in6m_free_list, in6_multi, in6m_nrele); + mtx_unlock(&in6_multi_free_mtx); + IN6_MULTI_LOCK(); + SLIST_FOREACH_SAFE(inm, &in6m_free_tmp, in6m_nrele, tinm) { + SLIST_REMOVE_HEAD(&in6m_free_tmp, in6m_nrele); + in6m_release(inm); + } + IN6_MULTI_UNLOCK(); } /* @@ -546,7 +666,7 @@ in6m_clear_recorded(struct in6_multi *inm) { struct ip6_msource *ims; - IN6_MULTI_LOCK_ASSERT(); + IN6_MULTI_LIST_LOCK_ASSERT(); RB_FOREACH(ims, ip6_msource_tree, &inm->in6m_srcs) { if (ims->im6s_stp) { @@ -586,7 +706,7 @@ in6m_record_source(struct in6_multi *inm, const struct in6_addr *addr) struct ip6_msource find; struct ip6_msource *ims, *nims; - IN6_MULTI_LOCK_ASSERT(); + IN6_MULTI_LIST_LOCK_ASSERT(); find.im6s_addr = *addr; ims = RB_FIND(ip6_msource_tree, &inm->in6m_srcs, &find); @@ -913,6 +1033,7 @@ in6m_merge(struct in6_multi *inm, /*const*/ struct in6_mfilter *imf) schanged = 0; error = 0; nsrc1 = nsrc0 = 0; + IN6_MULTI_LIST_LOCK_ASSERT(); /* * Update the source filters first, as this may fail. @@ -1089,65 +1210,16 @@ in6m_purge(struct in6_multi *inm) * * SMPng: Assume no mc locks held by caller. */ -struct in6_multi_mship * -in6_joingroup(struct ifnet *ifp, struct in6_addr *mcaddr, - int *errorp, int delay) -{ - struct in6_multi_mship *imm; - int error; - - imm = malloc(sizeof(*imm), M_IP6MADDR, M_NOWAIT); - if (imm == NULL) { - *errorp = ENOBUFS; - return (NULL); - } - - delay = (delay * PR_FASTHZ) / hz; - - error = in6_mc_join(ifp, mcaddr, NULL, &imm->i6mm_maddr, delay); - if (error) { - *errorp = error; - free(imm, M_IP6MADDR); - return (NULL); - } - - return (imm); -} - -/* - * Leave a multicast address w/o sources. - * KAME compatibility entry point. - * - * SMPng: Assume no mc locks held by caller. - */ -int -in6_leavegroup(struct in6_multi_mship *imm) -{ - - if (imm->i6mm_maddr != NULL) - in6_mc_leave(imm->i6mm_maddr, NULL); - free(imm, M_IP6MADDR); - return 0; -} - -/* - * Join a multicast group; unlocked entry point. - * - * SMPng: XXX: in6_mc_join() is called from in6_control() when upper - * locks are not held. Fortunately, ifp is unlikely to have been detached - * at this point, so we assume it's OK to recurse. - */ int -in6_mc_join(struct ifnet *ifp, const struct in6_addr *mcaddr, +in6_joingroup(struct ifnet *ifp, const struct in6_addr *mcaddr, /*const*/ struct in6_mfilter *imf, struct in6_multi **pinm, const int delay) { int error; IN6_MULTI_LOCK(); - error = in6_mc_join_locked(ifp, mcaddr, imf, pinm, delay); + error = in6_joingroup_locked(ifp, mcaddr, NULL, pinm, delay); IN6_MULTI_UNLOCK(); - return (error); } @@ -1161,12 +1233,13 @@ in6_mc_join(struct ifnet *ifp, const struct in6_addr *mcaddr, * code is returned. */ int -in6_mc_join_locked(struct ifnet *ifp, const struct in6_addr *mcaddr, +in6_joingroup_locked(struct ifnet *ifp, const struct in6_addr *mcaddr, /*const*/ struct in6_mfilter *imf, struct in6_multi **pinm, const int delay) { struct in6_mfilter timf; struct in6_multi *inm; + struct ifmultiaddr *ifma; int error; #ifdef KTR char ip6tbuf[INET6_ADDRSTRLEN]; @@ -1187,6 +1260,7 @@ in6_mc_join_locked(struct ifnet *ifp, const struct in6_addr *mcaddr, #endif IN6_MULTI_LOCK_ASSERT(); + IN6_MULTI_LIST_UNLOCK_ASSERT(); CTR4(KTR_MLD, "%s: join %s on %p(%s))", __func__, ip6_sprintf(ip6tbuf, mcaddr), ifp, if_name(ifp)); @@ -1202,13 +1276,13 @@ in6_mc_join_locked(struct ifnet *ifp, const struct in6_addr *mcaddr, im6f_init(&timf, MCAST_UNDEFINED, MCAST_EXCLUDE); imf = &timf; } - - error = in6_mc_get(ifp, mcaddr, &inm); + error = in6_getmulti(ifp, mcaddr, &inm); if (error) { - CTR1(KTR_MLD, "%s: in6_mc_get() failure", __func__); + CTR1(KTR_MLD, "%s: in6_getmulti() failure", __func__); return (error); } + IN6_MULTI_LIST_LOCK(); CTR1(KTR_MLD, "%s: merge inm state", __func__); error = in6m_merge(inm, imf); if (error) { @@ -1226,11 +1300,19 @@ in6_mc_join_locked(struct ifnet *ifp, const struct in6_addr *mcaddr, out_in6m_release: if (error) { CTR2(KTR_MLD, "%s: dropping ref on %p", __func__, inm); - in6m_release_locked(inm); + IF_ADDR_RLOCK(ifp); + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + if (ifma->ifma_protospec == inm) { + ifma->ifma_protospec = NULL; + break; + } + } + in6m_release_deferred(inm); + IF_ADDR_RUNLOCK(ifp); } else { *pinm = inm; } - + IN6_MULTI_LIST_UNLOCK(); return (error); } @@ -1238,14 +1320,13 @@ out_in6m_release: * Leave a multicast group; unlocked entry point. */ int -in6_mc_leave(struct in6_multi *inm, /*const*/ struct in6_mfilter *imf) +in6_leavegroup(struct in6_multi *inm, /*const*/ struct in6_mfilter *imf) { int error; IN6_MULTI_LOCK(); - error = in6_mc_leave_locked(inm, imf); + error = in6_leavegroup_locked(inm, imf); IN6_MULTI_UNLOCK(); - return (error); } @@ -1263,9 +1344,10 @@ in6_mc_leave(struct in6_multi *inm, /*const*/ struct in6_mfilter *imf) * makes a state change downcall into MLD. */ int -in6_mc_leave_locked(struct in6_multi *inm, /*const*/ struct in6_mfilter *imf) +in6_leavegroup_locked(struct in6_multi *inm, /*const*/ struct in6_mfilter *imf) { struct in6_mfilter timf; + struct ifnet *ifp; int error; #ifdef KTR char ip6tbuf[INET6_ADDRSTRLEN]; @@ -1296,6 +1378,9 @@ in6_mc_leave_locked(struct in6_multi *inm, /*const*/ struct in6_mfilter *imf) * to be allocated, and there is no opportunity to roll back * the transaction, it MUST NOT fail. */ + + ifp = inm->in6m_ifp; + IN6_MULTI_LIST_LOCK(); CTR1(KTR_MLD, "%s: merge inm state", __func__); error = in6m_merge(inm, imf); KASSERT(error == 0, ("%s: failed to merge inm state", __func__)); @@ -1306,11 +1391,17 @@ in6_mc_leave_locked(struct in6_multi *inm, /*const*/ struct in6_mfilter *imf) CTR1(KTR_MLD, "%s: failed mld downcall", __func__); CTR2(KTR_MLD, "%s: dropping ref on %p", __func__, inm); - in6m_release_locked(inm); + if (ifp) + IF_ADDR_WLOCK(ifp); + in6m_release_deferred(inm); + if (ifp) + IF_ADDR_WUNLOCK(ifp); + IN6_MULTI_LIST_UNLOCK(); return (error); } + /* * Block or unblock an ASM multicast source on an inpcb. * This implements the delta-based API described in RFC 3678. @@ -1448,8 +1539,7 @@ in6p_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) /* * Begin state merge transaction at MLD layer. */ - IN6_MULTI_LOCK(); - + IN6_MULTI_LIST_LOCK(); CTR1(KTR_MLD, "%s: merge inm state", __func__); error = in6m_merge(inm, imf); if (error) @@ -1461,7 +1551,7 @@ in6p_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) CTR1(KTR_MLD, "%s: failed mld downcall", __func__); } - IN6_MULTI_UNLOCK(); + IN6_MULTI_LIST_UNLOCK(); out_im6f_rollback: if (error) @@ -1530,22 +1620,36 @@ in6p_findmoptions(struct inpcb *inp) * Discard the IPv6 multicast options (and source filters). * * SMPng: NOTE: assumes INP write lock is held. + * + * XXX can all be safely deferred to epoch_call + * */ -void -ip6_freemoptions(struct ip6_moptions *imo) + +static void +inp_gcmoptions(epoch_context_t ctx) { + struct ip6_moptions *imo; struct in6_mfilter *imf; + struct in6_multi *inm; + struct ifnet *ifp; size_t idx, nmships; - KASSERT(imo != NULL, ("%s: ip6_moptions is NULL", __func__)); + imo = __containerof(ctx, struct ip6_moptions, imo6_epoch_ctx); nmships = imo->im6o_num_memberships; for (idx = 0; idx < nmships; ++idx) { imf = imo->im6o_mfilters ? &imo->im6o_mfilters[idx] : NULL; if (imf) im6f_leave(imf); - /* XXX this will thrash the lock(s) */ - (void)in6_mc_leave(imo->im6o_membership[idx], imf); + inm = imo->im6o_membership[idx]; + ifp = inm->in6m_ifp; + if (ifp != NULL) { + CURVNET_SET(ifp->if_vnet); + (void)in6_leavegroup(inm, imf); + CURVNET_RESTORE(); + } else { + (void)in6_leavegroup(inm, imf); + } if (imf) im6f_purge(imf); } @@ -1556,6 +1660,14 @@ ip6_freemoptions(struct ip6_moptions *imo) free(imo, M_IP6MOPTS); } +void +ip6_freemoptions(struct ip6_moptions *imo) +{ + if (imo == NULL) + return; + epoch_call(net_epoch_preempt, &imo->imo6_epoch_ctx, inp_gcmoptions); +} + /* * Atomically get source filters on a socket for an IPv6 multicast group. * Called with INP lock held; returns with lock released. @@ -2036,10 +2148,12 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) /* * Begin state merge transaction at MLD layer. */ + in_pcbref(inp); + INP_WUNLOCK(inp); IN6_MULTI_LOCK(); if (is_new) { - error = in6_mc_join_locked(ifp, &gsa->sin6.sin6_addr, imf, + error = in6_joingroup_locked(ifp, &gsa->sin6.sin6_addr, imf, &inm, 0); if (error) { IN6_MULTI_UNLOCK(); @@ -2048,6 +2162,7 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) imo->im6o_membership[idx] = inm; } else { CTR1(KTR_MLD, "%s: merge inm state", __func__); + IN6_MULTI_LIST_LOCK(); error = in6m_merge(inm, imf); if (error) CTR1(KTR_MLD, "%s: failed to merge inm state", @@ -2059,10 +2174,13 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) CTR1(KTR_MLD, "%s: failed mld downcall", __func__); } + IN6_MULTI_LIST_UNLOCK(); } IN6_MULTI_UNLOCK(); - INP_WLOCK_ASSERT(inp); + INP_WLOCK(inp); + if (in_pcbrele_wlocked(inp)) + return (ENXIO); if (error) { im6f_rollback(imf); if (is_new) @@ -2277,6 +2395,8 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) /* * Begin state merge transaction at MLD layer. */ + in_pcbref(inp); + INP_WUNLOCK(inp); IN6_MULTI_LOCK(); if (is_final) { @@ -2284,9 +2404,10 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) * Give up the multicast address record to which * the membership points. */ - (void)in6_mc_leave_locked(inm, imf); + (void)in6_leavegroup_locked(inm, imf); } else { CTR1(KTR_MLD, "%s: merge inm state", __func__); + IN6_MULTI_LIST_LOCK(); error = in6m_merge(inm, imf); if (error) CTR1(KTR_MLD, "%s: failed to merge inm state", @@ -2298,9 +2419,13 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) CTR1(KTR_MLD, "%s: failed mld downcall", __func__); } + IN6_MULTI_LIST_UNLOCK(); } IN6_MULTI_UNLOCK(); + INP_WLOCK(inp); + if (in_pcbrele_wlocked(inp)) + return (ENXIO); if (error) im6f_rollback(imf); @@ -2507,7 +2632,7 @@ in6p_set_source_filters(struct inpcb *inp, struct sockopt *sopt) goto out_im6f_rollback; INP_WLOCK_ASSERT(inp); - IN6_MULTI_LOCK(); + IN6_MULTI_LIST_LOCK(); /* * Begin state merge transaction at MLD layer. @@ -2523,7 +2648,7 @@ in6p_set_source_filters(struct inpcb *inp, struct sockopt *sopt) CTR1(KTR_MLD, "%s: failed mld downcall", __func__); } - IN6_MULTI_UNLOCK(); + IN6_MULTI_LIST_UNLOCK(); out_im6f_rollback: if (error) @@ -2714,9 +2839,9 @@ sysctl_ip6_mcast_filters(SYSCTL_HANDLER_ARGS) return (retval); IN6_MULTI_LOCK(); - + IN6_MULTI_LIST_LOCK(); IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET6 || ifma->ifma_protospec == NULL) continue; @@ -2746,6 +2871,7 @@ sysctl_ip6_mcast_filters(SYSCTL_HANDLER_ARGS) } IF_ADDR_RUNLOCK(ifp); + IN6_MULTI_LIST_UNLOCK(); IN6_MULTI_UNLOCK(); return (retval); diff --git a/freebsd/sys/netinet6/in6_pcb.c b/freebsd/sys/netinet6/in6_pcb.c index a4bbd6a9..488cca86 100644 --- a/freebsd/sys/netinet6/in6_pcb.c +++ b/freebsd/sys/netinet6/in6_pcb.c @@ -134,7 +134,7 @@ in6_pcbbind(struct inpcb *inp, struct sockaddr *nam, INP_WLOCK_ASSERT(inp); INP_HASH_WLOCK_ASSERT(pcbinfo); - if (TAILQ_EMPTY(&V_in6_ifaddrhead)) /* XXX broken! */ + if (CK_STAILQ_EMPTY(&V_in6_ifaddrhead)) /* XXX broken! */ return (EADDRNOTAVAIL); if (inp->inp_lport || !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) return (EINVAL); @@ -176,9 +176,11 @@ in6_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct ifaddr *ifa; sin6->sin6_port = 0; /* yech... */ + NET_EPOCH_ENTER(); if ((ifa = ifa_ifwithaddr((struct sockaddr *)sin6)) == NULL && (inp->inp_flags & INP_BINDANY) == 0) { + NET_EPOCH_EXIT(); return (EADDRNOTAVAIL); } @@ -191,11 +193,10 @@ in6_pcbbind(struct inpcb *inp, struct sockaddr *nam, if (ifa != NULL && ((struct in6_ifaddr *)ifa)->ia6_flags & (IN6_IFF_ANYCAST|IN6_IFF_NOTREADY|IN6_IFF_DETACHED)) { - ifa_free(ifa); + NET_EPOCH_EXIT(); return (EADDRNOTAVAIL); } - if (ifa != NULL) - ifa_free(ifa); + NET_EPOCH_EXIT(); } if (lport) { struct inpcb *t; @@ -363,7 +364,7 @@ in6_pcbladdr(struct inpcb *inp, struct sockaddr *nam, if ((error = sa6_embedscope(sin6, V_ip6_use_defzone)) != 0) return(error); - if (!TAILQ_EMPTY(&V_in6_ifaddrhead)) { + if (!CK_STAILQ_EMPTY(&V_in6_ifaddrhead)) { /* * If the destination address is UNSPECIFIED addr, * use the loopback addr, e.g ::1. @@ -819,8 +820,7 @@ in6_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp) for (i = 0; i < im6o->im6o_num_memberships; i++) { if (im6o->im6o_membership[i]->in6m_ifp == ifp) { - in6_mc_leave(im6o->im6o_membership[i], - NULL); + in6_leavegroup(im6o->im6o_membership[i], NULL); gap++; } else if (gap != 0) { im6o->im6o_membership[i - gap] = diff --git a/freebsd/sys/netinet6/in6_proto.c b/freebsd/sys/netinet6/in6_proto.c index 55fbbe80..756ea48b 100644 --- a/freebsd/sys/netinet6/in6_proto.c +++ b/freebsd/sys/netinet6/in6_proto.c @@ -284,7 +284,6 @@ struct protosw inet6sw[] = { .pr_input = encap6_input, .pr_output = rip6_output, .pr_ctloutput = rip6_ctloutput, - .pr_init = encap_init, .pr_usrreqs = &rip6_usrreqs }, #endif /* INET */ @@ -296,7 +295,6 @@ struct protosw inet6sw[] = { .pr_input = encap6_input, .pr_output = rip6_output, .pr_ctloutput = rip6_ctloutput, - .pr_init = encap_init, .pr_usrreqs = &rip6_usrreqs }, { @@ -307,7 +305,6 @@ struct protosw inet6sw[] = { .pr_input = encap6_input, .pr_output = rip6_output, .pr_ctloutput = rip6_ctloutput, - .pr_init = encap_init, .pr_usrreqs = &rip6_usrreqs }, { diff --git a/freebsd/sys/netinet6/in6_src.c b/freebsd/sys/netinet6/in6_src.c index 5b110274..92f7df4e 100644 --- a/freebsd/sys/netinet6/in6_src.c +++ b/freebsd/sys/netinet6/in6_src.c @@ -311,7 +311,7 @@ in6_selectsrc(uint32_t fibnum, struct sockaddr_in6 *dstsock, return (error); IN6_IFADDR_RLOCK(&in6_ifa_tracker); - TAILQ_FOREACH(ia, &V_in6_ifaddrhead, ia_link) { + CK_STAILQ_FOREACH(ia, &V_in6_ifaddrhead, ia_link) { int new_scope = -1, new_matchlen = -1; struct in6_addrpolicy *new_policy = NULL; u_int32_t srczone, osrczone, dstzone; diff --git a/freebsd/sys/netinet6/in6_var.h b/freebsd/sys/netinet6/in6_var.h index 2e22d962..6b4fe1ab 100644 --- a/freebsd/sys/netinet6/in6_var.h +++ b/freebsd/sys/netinet6/in6_var.h @@ -100,6 +100,7 @@ struct nd_ifinfo; struct scope6_id; struct lltable; struct mld_ifsoftc; +struct in6_multi; struct in6_ifextra { counter_u64_t *in6_ifstat; @@ -113,6 +114,10 @@ struct in6_ifextra { #define LLTABLE6(ifp) (((struct in6_ifextra *)(ifp)->if_afdata[AF_INET6])->lltable) #ifdef _KERNEL + +SLIST_HEAD(in6_multi_head, in6_multi); +MALLOC_DECLARE(M_IP6MADDR); + struct in6_ifaddr { struct ifaddr ia_ifa; /* protocol-independent info */ #define ia_ifp ia_ifa.ifa_ifp @@ -122,7 +127,7 @@ struct in6_ifaddr { struct sockaddr_in6 ia_dstaddr; /* space for destination addr */ struct sockaddr_in6 ia_prefixmask; /* prefix mask */ u_int32_t ia_plen; /* prefix length */ - TAILQ_ENTRY(in6_ifaddr) ia_link; /* list of IPv6 addresses */ + CK_STAILQ_ENTRY(in6_ifaddr) ia_link; /* list of IPv6 addresses */ int ia6_flags; struct in6_addrlifetime ia6_lifetime; @@ -137,12 +142,12 @@ struct in6_ifaddr { /* multicast addresses joined from the kernel */ LIST_HEAD(, in6_multi_mship) ia6_memberships; /* entry in bucket of inet6 addresses */ - LIST_ENTRY(in6_ifaddr) ia6_hash; + CK_LIST_ENTRY(in6_ifaddr) ia6_hash; }; /* List of in6_ifaddr's. */ -TAILQ_HEAD(in6_ifaddrhead, in6_ifaddr); -LIST_HEAD(in6_ifaddrlisthead, in6_ifaddr); +CK_STAILQ_HEAD(in6_ifaddrhead, in6_ifaddr); +CK_LIST_HEAD(in6_ifaddrlisthead, in6_ifaddr); #endif /* _KERNEL */ /* control structure to manage address selection policy */ @@ -630,7 +635,6 @@ struct in6_multi_mship { * w/o breaking the ABI for ifmcstat. */ struct in6_multi { - LIST_ENTRY(in6_multi) in6m_entry; /* list glue */ struct in6_addr in6m_addr; /* IPv6 multicast address */ struct ifnet *in6m_ifp; /* back pointer to ifnet */ struct ifmultiaddr *in6m_ifma; /* back pointer to ifmultiaddr */ @@ -666,6 +670,8 @@ struct in6_multi { } in6m_st[2]; /* state at t0, t1 */ }; +void in6m_disconnect(struct in6_multi *inm); +extern int ifma6_restart; /* * Helper function to derive the filter mode on a source entry * from its internal counters. Predicates are: @@ -694,11 +700,19 @@ im6s_get_mode(const struct in6_multi *inm, const struct ip6_msource *ims, * consumers of IN_*_MULTI() macros should acquire the locks before * calling them; users of the in_{add,del}multi() functions should not. */ -extern struct mtx in6_multi_mtx; -#define IN6_MULTI_LOCK() mtx_lock(&in6_multi_mtx) -#define IN6_MULTI_UNLOCK() mtx_unlock(&in6_multi_mtx) -#define IN6_MULTI_LOCK_ASSERT() mtx_assert(&in6_multi_mtx, MA_OWNED) -#define IN6_MULTI_UNLOCK_ASSERT() mtx_assert(&in6_multi_mtx, MA_NOTOWNED) +extern struct mtx in6_multi_list_mtx; +extern struct sx in6_multi_sx; + +#define IN6_MULTI_LIST_LOCK() mtx_lock(&in6_multi_list_mtx) +#define IN6_MULTI_LIST_UNLOCK() mtx_unlock(&in6_multi_list_mtx) +#define IN6_MULTI_LIST_LOCK_ASSERT() mtx_assert(&in6_multi_list_mtx, MA_OWNED) +#define IN6_MULTI_LIST_UNLOCK_ASSERT() mtx_assert(&in6_multi_list_mtx, MA_NOTOWNED) + +#define IN6_MULTI_LOCK() sx_xlock(&in6_multi_sx) +#define IN6_MULTI_UNLOCK() sx_xunlock(&in6_multi_sx) +#define IN6_MULTI_LOCK_ASSERT() sx_assert(&in6_multi_sx, SA_XLOCKED) +#define IN6_MULTI_UNLOCK_ASSERT() sx_assert(&in6_multi_sx, SA_XUNLOCKED) + /* * Look up an in6_multi record for an IPv6 multicast address @@ -713,13 +727,12 @@ in6m_lookup_locked(struct ifnet *ifp, const struct in6_addr *mcaddr) struct ifmultiaddr *ifma; struct in6_multi *inm; - IN6_MULTI_LOCK_ASSERT(); - IF_ADDR_LOCK_ASSERT(ifp); - inm = NULL; - TAILQ_FOREACH(ifma, &((ifp)->if_multiaddrs), ifma_link) { + CK_STAILQ_FOREACH(ifma, &((ifp)->if_multiaddrs), ifma_link) { if (ifma->ifma_addr->sa_family == AF_INET6) { inm = (struct in6_multi *)ifma->ifma_protospec; + if (inm == NULL) + continue; if (IN6_ARE_ADDR_EQUAL(&inm->in6m_addr, mcaddr)) break; inm = NULL; @@ -738,11 +751,11 @@ in6m_lookup(struct ifnet *ifp, const struct in6_addr *mcaddr) { struct in6_multi *inm; - IN6_MULTI_LOCK(); + IN6_MULTI_LIST_LOCK(); IF_ADDR_RLOCK(ifp); inm = in6m_lookup_locked(ifp, mcaddr); IF_ADDR_RUNLOCK(ifp); - IN6_MULTI_UNLOCK(); + IN6_MULTI_LIST_UNLOCK(); return (inm); } @@ -752,36 +765,55 @@ static __inline void in6m_acquire_locked(struct in6_multi *inm) { - IN6_MULTI_LOCK_ASSERT(); + IN6_MULTI_LIST_LOCK_ASSERT(); ++inm->in6m_refcount; } +static __inline void +in6m_acquire(struct in6_multi *inm) +{ + IN6_MULTI_LIST_LOCK(); + in6m_acquire_locked(inm); + IN6_MULTI_LIST_UNLOCK(); +} + +static __inline void +in6m_rele_locked(struct in6_multi_head *inmh, struct in6_multi *inm) +{ + KASSERT(inm->in6m_refcount > 0, ("refcount == %d inm: %p", inm->in6m_refcount, inm)); + IN6_MULTI_LIST_LOCK_ASSERT(); + + if (--inm->in6m_refcount == 0) { + in6m_disconnect(inm); + inm->in6m_ifma->ifma_protospec = NULL; + MPASS(inm->in6m_ifma->ifma_llifma == NULL); + SLIST_INSERT_HEAD(inmh, inm, in6m_nrele); + } +} + struct ip6_moptions; struct sockopt; +struct inpcbinfo; /* Multicast KPIs. */ int im6o_mc_filter(const struct ip6_moptions *, const struct ifnet *, const struct sockaddr *, const struct sockaddr *); -int in6_mc_join(struct ifnet *, const struct in6_addr *, +int in6_joingroup(struct ifnet *, const struct in6_addr *, struct in6_mfilter *, struct in6_multi **, int); -int in6_mc_join_locked(struct ifnet *, const struct in6_addr *, +int in6_joingroup_locked(struct ifnet *, const struct in6_addr *, struct in6_mfilter *, struct in6_multi **, int); -int in6_mc_leave(struct in6_multi *, struct in6_mfilter *); -int in6_mc_leave_locked(struct in6_multi *, struct in6_mfilter *); +int in6_leavegroup(struct in6_multi *, struct in6_mfilter *); +int in6_leavegroup_locked(struct in6_multi *, struct in6_mfilter *); void in6m_clear_recorded(struct in6_multi *); void in6m_commit(struct in6_multi *); void in6m_print(const struct in6_multi *); int in6m_record_source(struct in6_multi *, const struct in6_addr *); -void in6m_release_locked(struct in6_multi *); +void in6m_release_deferred(struct in6_multi *); +void in6m_release_list_deferred(struct in6_multi_head *); void ip6_freemoptions(struct ip6_moptions *); int ip6_getmoptions(struct inpcb *, struct sockopt *); int ip6_setmoptions(struct inpcb *, struct sockopt *); -/* Legacy KAME multicast KPIs. */ -struct in6_multi_mship * - in6_joingroup(struct ifnet *, struct in6_addr *, int *, int); -int in6_leavegroup(struct in6_multi_mship *); - /* flags to in6_update_ifa */ #define IN6_IFAUPDATE_DADDELAY 0x1 /* first time to configure an address */ diff --git a/freebsd/sys/netinet6/ip6_fastfwd.c b/freebsd/sys/netinet6/ip6_fastfwd.c index 056f9315..f63c51bf 100644 --- a/freebsd/sys/netinet6/ip6_fastfwd.c +++ b/freebsd/sys/netinet6/ip6_fastfwd.c @@ -99,7 +99,8 @@ ip6_tryforward(struct mbuf *m) * Fallback conditions to ip6_input for slow path processing. */ ip6 = mtod(m, struct ip6_hdr *); - if (ip6->ip6_nxt == IPPROTO_HOPOPTS || + if ((m->m_flags & (M_BCAST | M_MCAST)) != 0 || + ip6->ip6_nxt == IPPROTO_HOPOPTS || IN6_IS_ADDR_MULTICAST(&ip6->ip6_dst) || IN6_IS_ADDR_LINKLOCAL(&ip6->ip6_dst) || IN6_IS_ADDR_LINKLOCAL(&ip6->ip6_src) || @@ -196,12 +197,19 @@ passin: in6_ifstat_inc(rcvif, ifs6_in_noroute); goto dropin; } + if (!PFIL_HOOKED(&V_inet6_pfil_hook)) { + if (m->m_pkthdr.len > nh.nh_mtu) { + in6_ifstat_inc(nh.nh_ifp, ifs6_in_toobig); + icmp6_error(m, ICMP6_PACKET_TOO_BIG, 0, nh.nh_mtu); + m = NULL; + goto dropout; + } + goto passout; + } /* * Outgoing packet firewall processing. */ - if (!PFIL_HOOKED(&V_inet6_pfil_hook)) - goto passout; if (pfil_run_hooks(&V_inet6_pfil_hook, &m, nh.nh_ifp, PFIL_OUT, PFIL_FWD, NULL) != 0 || m == NULL) goto dropout; diff --git a/freebsd/sys/netinet6/ip6_input.c b/freebsd/sys/netinet6/ip6_input.c index 7e1007e3..77e32da8 100644 --- a/freebsd/sys/netinet6/ip6_input.c +++ b/freebsd/sys/netinet6/ip6_input.c @@ -224,7 +224,7 @@ ip6_init(void) TUNABLE_INT_FETCH("net.inet6.ip6.accept_rtadv", &V_ip6_accept_rtadv); TUNABLE_INT_FETCH("net.inet6.ip6.no_radr", &V_ip6_no_radr); - TAILQ_INIT(&V_in6_ifaddrhead); + CK_STAILQ_INIT(&V_in6_ifaddrhead); V_in6_ifaddrhashtbl = hashinit(IN6ADDR_NHASH, M_IFADDR, &V_in6_ifaddrhmask); @@ -379,10 +379,10 @@ ip6_destroy(void *unused __unused) /* Cleanup addresses. */ IFNET_RLOCK(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { /* Cannot lock here - lock recursion. */ /* IF_ADDR_LOCK(ifp); */ - TAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, nifa) { + CK_STAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, nifa) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; diff --git a/freebsd/sys/netinet6/ip6_var.h b/freebsd/sys/netinet6/ip6_var.h index 42c235d1..74b5f89c 100644 --- a/freebsd/sys/netinet6/ip6_var.h +++ b/freebsd/sys/netinet6/ip6_var.h @@ -66,6 +66,8 @@ #ifndef _NETINET6_IP6_VAR_H_ #define _NETINET6_IP6_VAR_H_ +#include <sys/epoch.h> + /* * IP6 reassembly queue structure. Each fragment * being reassembled is attached to one of these structures. @@ -121,6 +123,7 @@ struct ip6_moptions { u_short im6o_max_memberships; /* max memberships this socket */ struct in6_multi **im6o_membership; /* group memberships */ struct in6_mfilter *im6o_mfilters; /* source filters */ + struct epoch_context imo6_epoch_ctx; }; /* diff --git a/freebsd/sys/netinet6/mld6.c b/freebsd/sys/netinet6/mld6.c index c1dff0c3..0c82d5ff 100644 --- a/freebsd/sys/netinet6/mld6.c +++ b/freebsd/sys/netinet6/mld6.c @@ -126,7 +126,7 @@ static int mld_v1_input_query(struct ifnet *, const struct ip6_hdr *, /*const*/ struct mld_hdr *); static int mld_v1_input_report(struct ifnet *, const struct ip6_hdr *, /*const*/ struct mld_hdr *); -static void mld_v1_process_group_timer(struct mld_ifsoftc *, +static void mld_v1_process_group_timer(struct in6_multi_head *, struct in6_multi *); static void mld_v1_process_querier_timers(struct mld_ifsoftc *); static int mld_v1_transmit_report(struct in6_multi *, const int); @@ -144,7 +144,7 @@ static int mld_v2_input_query(struct ifnet *, const struct ip6_hdr *, struct mbuf *, const int, const int); static int mld_v2_merge_state_changes(struct in6_multi *, struct mbufq *); -static void mld_v2_process_group_timers(struct mld_ifsoftc *, +static void mld_v2_process_group_timers(struct in6_multi_head *, struct mbufq *, struct mbufq *, struct in6_multi *, const int); static int mld_v2_process_group_query(struct in6_multi *, @@ -379,6 +379,7 @@ sysctl_mld_ifinfo(SYSCTL_HANDLER_ARGS) return (error); IN6_MULTI_LOCK(); + IN6_MULTI_LIST_LOCK(); MLD_LOCK(); if (name[0] <= 0 || name[0] > V_if_index) { @@ -411,6 +412,7 @@ sysctl_mld_ifinfo(SYSCTL_HANDLER_ARGS) out_locked: MLD_UNLOCK(); + IN6_MULTI_LIST_UNLOCK(); IN6_MULTI_UNLOCK(); return (error); } @@ -510,7 +512,6 @@ mli_alloc_locked(/*const*/ struct ifnet *ifp) mli->mli_qi = MLD_QI_INIT; mli->mli_qri = MLD_QRI_INIT; mli->mli_uri = MLD_URI_INIT; - SLIST_INIT(&mli->mli_relinmhead); mbufq_init(&mli->mli_gq, MLD_MAX_RESPONSE_PACKETS); LIST_INSERT_HEAD(&V_mli_head, mli, mli_link); @@ -537,38 +538,41 @@ void mld_ifdetach(struct ifnet *ifp) { struct mld_ifsoftc *mli; - struct ifmultiaddr *ifma; - struct in6_multi *inm, *tinm; + struct ifmultiaddr *ifma, *next; + struct in6_multi *inm; + struct in6_multi_head inmh; CTR3(KTR_MLD, "%s: called for ifp %p(%s)", __func__, ifp, if_name(ifp)); - IN6_MULTI_LOCK_ASSERT(); + SLIST_INIT(&inmh); + IN6_MULTI_LIST_LOCK_ASSERT(); MLD_LOCK(); mli = MLD_IFINFO(ifp); if (mli->mli_version == MLD_VERSION_2) { - IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + IF_ADDR_WLOCK(ifp); + restart: + CK_STAILQ_FOREACH_SAFE(ifma, &ifp->if_multiaddrs, ifma_link, next) { if (ifma->ifma_addr->sa_family != AF_INET6 || ifma->ifma_protospec == NULL) continue; inm = (struct in6_multi *)ifma->ifma_protospec; if (inm->in6m_state == MLD_LEAVING_MEMBER) { - SLIST_INSERT_HEAD(&mli->mli_relinmhead, - inm, in6m_nrele); + in6m_rele_locked(&inmh, inm); + ifma->ifma_protospec = NULL; } in6m_clear_recorded(inm); + if (__predict_false(ifma6_restart)) { + ifma6_restart = false; + goto restart; + } } - IF_ADDR_RUNLOCK(ifp); - SLIST_FOREACH_SAFE(inm, &mli->mli_relinmhead, in6m_nrele, - tinm) { - SLIST_REMOVE_HEAD(&mli->mli_relinmhead, in6m_nrele); - in6m_release_locked(inm); - } + IF_ADDR_WUNLOCK(ifp); } MLD_UNLOCK(); + in6m_release_list_deferred(&inmh); } /* @@ -608,10 +612,6 @@ mli_delete_locked(const struct ifnet *ifp) LIST_REMOVE(mli, mli_link); - KASSERT(SLIST_EMPTY(&mli->mli_relinmhead), - ("%s: there are dangling in_multi references", - __func__)); - free(mli, M_MLD); return; } @@ -682,7 +682,7 @@ mld_v1_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, in6_setscope(&mld->mld_addr, ifp, NULL); } - IN6_MULTI_LOCK(); + IN6_MULTI_LIST_LOCK(); MLD_LOCK(); /* @@ -703,8 +703,8 @@ mld_v1_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, * interface, kick the report timer. */ CTR2(KTR_MLD, "process v1 general query on ifp %p(%s)", - ifp, if_name(ifp)); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + ifp, if_name(ifp)); + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET6 || ifma->ifma_protospec == NULL) continue; @@ -730,7 +730,7 @@ mld_v1_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, IF_ADDR_RUNLOCK(ifp); MLD_UNLOCK(); - IN6_MULTI_UNLOCK(); + IN6_MULTI_LIST_UNLOCK(); return (0); } @@ -761,7 +761,7 @@ mld_v1_update_group(struct in6_multi *inm, const int timer) ip6_sprintf(ip6tbuf, &inm->in6m_addr), if_name(inm->in6m_ifp), timer); - IN6_MULTI_LOCK_ASSERT(); + IN6_MULTI_LIST_LOCK_ASSERT(); switch (inm->in6m_state) { case MLD_NOT_MEMBER: @@ -884,7 +884,7 @@ mld_v2_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, in6_setscope(&mld->mld_addr, ifp, NULL); } - IN6_MULTI_LOCK(); + IN6_MULTI_LIST_LOCK(); MLD_LOCK(); mli = MLD_IFINFO(ifp); @@ -967,7 +967,7 @@ mld_v2_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, out_locked: MLD_UNLOCK(); - IN6_MULTI_UNLOCK(); + IN6_MULTI_LIST_UNLOCK(); return (0); } @@ -985,7 +985,7 @@ mld_v2_process_group_query(struct in6_multi *inm, struct mld_ifsoftc *mli, int retval; uint16_t nsrc; - IN6_MULTI_LOCK_ASSERT(); + IN6_MULTI_LIST_LOCK_ASSERT(); MLD_LOCK_ASSERT(); retval = 0; @@ -1170,7 +1170,7 @@ mld_v1_input_report(struct ifnet *ifp, const struct ip6_hdr *ip6, if (!IN6_IS_ADDR_UNSPECIFIED(&mld->mld_addr)) in6_setscope(&mld->mld_addr, ifp, NULL); - IN6_MULTI_LOCK(); + IN6_MULTI_LIST_LOCK(); MLD_LOCK(); IF_ADDR_RLOCK(ifp); @@ -1222,7 +1222,7 @@ mld_v1_input_report(struct ifnet *ifp, const struct ip6_hdr *ip6, out_locked: IF_ADDR_RUNLOCK(ifp); MLD_UNLOCK(); - IN6_MULTI_UNLOCK(); + IN6_MULTI_LIST_UNLOCK(); /* XXX Clear embedded scope ID as userland won't expect it. */ in6_clearscope(&mld->mld_addr); @@ -1333,8 +1333,9 @@ mld_fasttimo_vnet(void) struct mbufq qrq; /* Query response packets */ struct ifnet *ifp; struct mld_ifsoftc *mli; - struct ifmultiaddr *ifma; + struct ifmultiaddr *ifma, *next; struct in6_multi *inm, *tinm; + struct in6_multi_head inmh; int uri_fasthz; uri_fasthz = 0; @@ -1349,7 +1350,8 @@ mld_fasttimo_vnet(void) !V_state_change_timers_running6) return; - IN6_MULTI_LOCK(); + SLIST_INIT(&inmh); + IN6_MULTI_LIST_LOCK(); MLD_LOCK(); /* @@ -1393,23 +1395,28 @@ mld_fasttimo_vnet(void) mbufq_init(&scq, MLD_MAX_STATE_CHANGE_PACKETS); } - IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + IF_ADDR_WLOCK(ifp); + restart: + CK_STAILQ_FOREACH_SAFE(ifma, &ifp->if_multiaddrs, ifma_link, next) { if (ifma->ifma_addr->sa_family != AF_INET6 || ifma->ifma_protospec == NULL) continue; inm = (struct in6_multi *)ifma->ifma_protospec; switch (mli->mli_version) { case MLD_VERSION_1: - mld_v1_process_group_timer(mli, inm); + mld_v1_process_group_timer(&inmh, inm); break; case MLD_VERSION_2: - mld_v2_process_group_timers(mli, &qrq, + mld_v2_process_group_timers(&inmh, &qrq, &scq, inm, uri_fasthz); break; } + if (__predict_false(ifma6_restart)) { + ifma6_restart = false; + goto restart; + } } - IF_ADDR_RUNLOCK(ifp); + IF_ADDR_WUNLOCK(ifp); switch (mli->mli_version) { case MLD_VERSION_1: @@ -1421,9 +1428,8 @@ mld_fasttimo_vnet(void) * IF_ADDR_LOCK internally as well as * ip6_output() to transmit a packet. */ - SLIST_FOREACH_SAFE(inm, &mli->mli_relinmhead, - in6m_nrele, tinm) { - SLIST_REMOVE_HEAD(&mli->mli_relinmhead, + SLIST_FOREACH_SAFE(inm, &inmh, in6m_nrele, tinm) { + SLIST_REMOVE_HEAD(&inmh, in6m_nrele); (void)mld_v1_transmit_report(inm, MLD_LISTENER_REPORT); @@ -1437,19 +1443,14 @@ mld_fasttimo_vnet(void) * Free the in_multi reference(s) for * this lifecycle. */ - SLIST_FOREACH_SAFE(inm, &mli->mli_relinmhead, - in6m_nrele, tinm) { - SLIST_REMOVE_HEAD(&mli->mli_relinmhead, - in6m_nrele); - in6m_release_locked(inm); - } + in6m_release_list_deferred(&inmh); break; } } out_locked: MLD_UNLOCK(); - IN6_MULTI_UNLOCK(); + IN6_MULTI_LIST_UNLOCK(); } /* @@ -1457,11 +1458,11 @@ out_locked: * Will update the global pending timer flags. */ static void -mld_v1_process_group_timer(struct mld_ifsoftc *mli, struct in6_multi *inm) +mld_v1_process_group_timer(struct in6_multi_head *inmh, struct in6_multi *inm) { int report_timer_expired; - IN6_MULTI_LOCK_ASSERT(); + IN6_MULTI_LIST_LOCK_ASSERT(); MLD_LOCK_ASSERT(); if (inm->in6m_timer == 0) { @@ -1484,8 +1485,7 @@ mld_v1_process_group_timer(struct mld_ifsoftc *mli, struct in6_multi *inm) case MLD_REPORTING_MEMBER: if (report_timer_expired) { inm->in6m_state = MLD_IDLE_MEMBER; - SLIST_INSERT_HEAD(&mli->mli_relinmhead, inm, - in6m_nrele); + in6m_rele_locked(inmh, inm); } break; case MLD_G_QUERY_PENDING_MEMBER: @@ -1501,7 +1501,7 @@ mld_v1_process_group_timer(struct mld_ifsoftc *mli, struct in6_multi *inm) * Note: Unlocked read from mli. */ static void -mld_v2_process_group_timers(struct mld_ifsoftc *mli, +mld_v2_process_group_timers(struct in6_multi_head *inmh, struct mbufq *qrq, struct mbufq *scq, struct in6_multi *inm, const int uri_fasthz) { @@ -1511,7 +1511,7 @@ mld_v2_process_group_timers(struct mld_ifsoftc *mli, char ip6tbuf[INET6_ADDRSTRLEN]; #endif - IN6_MULTI_LOCK_ASSERT(); + IN6_MULTI_LIST_LOCK_ASSERT(); MLD_LOCK_ASSERT(); query_response_timer_expired = 0; @@ -1609,8 +1609,7 @@ mld_v2_process_group_timers(struct mld_ifsoftc *mli, if (inm->in6m_state == MLD_LEAVING_MEMBER && inm->in6m_scrv == 0) { inm->in6m_state = MLD_NOT_MEMBER; - SLIST_INSERT_HEAD(&mli->mli_relinmhead, - inm, in6m_nrele); + in6m_rele_locked(inmh, inm); } } break; @@ -1654,14 +1653,16 @@ mld_set_version(struct mld_ifsoftc *mli, const int version) static void mld_v2_cancel_link_timers(struct mld_ifsoftc *mli) { - struct ifmultiaddr *ifma; + struct ifmultiaddr *ifma, *next; struct ifnet *ifp; - struct in6_multi *inm, *tinm; + struct in6_multi *inm; + struct in6_multi_head inmh; CTR3(KTR_MLD, "%s: cancel v2 timers on ifp %p(%s)", __func__, mli->mli_ifp, if_name(mli->mli_ifp)); - IN6_MULTI_LOCK_ASSERT(); + SLIST_INIT(&inmh); + IN6_MULTI_LIST_LOCK_ASSERT(); MLD_LOCK_ASSERT(); /* @@ -1677,8 +1678,9 @@ mld_v2_cancel_link_timers(struct mld_ifsoftc *mli) ifp = mli->mli_ifp; - IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + IF_ADDR_WLOCK(ifp); + restart: + CK_STAILQ_FOREACH_SAFE(ifma, &ifp->if_multiaddrs, ifma_link, next) { if (ifma->ifma_addr->sa_family != AF_INET6) continue; inm = (struct in6_multi *)ifma->ifma_protospec; @@ -1696,8 +1698,8 @@ mld_v2_cancel_link_timers(struct mld_ifsoftc *mli) * version, we need to release the final * reference held for issuing the INCLUDE {}. */ - SLIST_INSERT_HEAD(&mli->mli_relinmhead, inm, - in6m_nrele); + in6m_rele_locked(&inmh, inm); + ifma->ifma_protospec = NULL; /* FALLTHROUGH */ case MLD_G_QUERY_PENDING_MEMBER: case MLD_SG_QUERY_PENDING_MEMBER: @@ -1713,12 +1715,13 @@ mld_v2_cancel_link_timers(struct mld_ifsoftc *mli) mbufq_drain(&inm->in6m_scq); break; } + if (__predict_false(ifma6_restart)) { + ifma6_restart = false; + goto restart; + } } - IF_ADDR_RUNLOCK(ifp); - SLIST_FOREACH_SAFE(inm, &mli->mli_relinmhead, in6m_nrele, tinm) { - SLIST_REMOVE_HEAD(&mli->mli_relinmhead, in6m_nrele); - in6m_release_locked(inm); - } + IF_ADDR_WUNLOCK(ifp); + in6m_release_list_deferred(&inmh); } /* @@ -1790,7 +1793,7 @@ mld_v1_transmit_report(struct in6_multi *in6m, const int type) struct mbuf *mh, *md; struct mld_hdr *mld; - IN6_MULTI_LOCK_ASSERT(); + IN6_MULTI_LIST_LOCK_ASSERT(); MLD_LOCK_ASSERT(); ifp = in6m->in6m_ifp; @@ -1881,7 +1884,7 @@ mld_change_state(struct in6_multi *inm, const int delay) struct ifnet *ifp; int error; - IN6_MULTI_LOCK_ASSERT(); + IN6_MULTI_LIST_LOCK_ASSERT(); error = 0; @@ -1965,7 +1968,7 @@ mld_initial_join(struct in6_multi *inm, struct mld_ifsoftc *mli, ifp = inm->in6m_ifp; - IN6_MULTI_LOCK_ASSERT(); + IN6_MULTI_LIST_LOCK_ASSERT(); MLD_LOCK_ASSERT(); KASSERT(mli && mli->mli_ifp == ifp, ("%s: inconsistent ifp", __func__)); @@ -1995,7 +1998,7 @@ mld_initial_join(struct in6_multi *inm, struct mld_ifsoftc *mli, */ if (mli->mli_version == MLD_VERSION_2 && inm->in6m_state == MLD_LEAVING_MEMBER) - in6m_release_locked(inm); + in6m_release_deferred(inm); inm->in6m_state = MLD_REPORTING_MEMBER; @@ -2108,7 +2111,7 @@ mld_handle_state_change(struct in6_multi *inm, struct mld_ifsoftc *mli) ifp = inm->in6m_ifp; - IN6_MULTI_LOCK_ASSERT(); + IN6_MULTI_LIST_LOCK_ASSERT(); MLD_LOCK_ASSERT(); KASSERT(mli && mli->mli_ifp == ifp, @@ -2171,7 +2174,7 @@ mld_final_leave(struct in6_multi *inm, struct mld_ifsoftc *mli) __func__, ip6_sprintf(ip6tbuf, &inm->in6m_addr), inm->in6m_ifp, if_name(inm->in6m_ifp)); - IN6_MULTI_LOCK_ASSERT(); + IN6_MULTI_LIST_LOCK_ASSERT(); MLD_LOCK_ASSERT(); switch (inm->in6m_state) { @@ -2298,7 +2301,7 @@ mld_v2_enqueue_group_record(struct mbufq *mq, struct in6_multi *inm, char ip6tbuf[INET6_ADDRSTRLEN]; #endif - IN6_MULTI_LOCK_ASSERT(); + IN6_MULTI_LIST_LOCK_ASSERT(); ifp = inm->in6m_ifp; is_filter_list_change = 0; @@ -2681,7 +2684,7 @@ mld_v2_enqueue_filter_change(struct mbufq *mq, struct in6_multi *inm) char ip6tbuf[INET6_ADDRSTRLEN]; #endif - IN6_MULTI_LOCK_ASSERT(); + IN6_MULTI_LIST_LOCK_ASSERT(); if (inm->in6m_nsrc == 0 || (inm->in6m_st[0].iss_asm > 0 && inm->in6m_st[1].iss_asm > 0)) @@ -2881,7 +2884,7 @@ mld_v2_merge_state_changes(struct in6_multi *inm, struct mbufq *scq) domerge = 0; recslen = 0; - IN6_MULTI_LOCK_ASSERT(); + IN6_MULTI_LIST_LOCK_ASSERT(); MLD_LOCK_ASSERT(); /* @@ -2980,7 +2983,7 @@ mld_v2_dispatch_general_query(struct mld_ifsoftc *mli) struct in6_multi *inm; int retval; - IN6_MULTI_LOCK_ASSERT(); + IN6_MULTI_LIST_LOCK_ASSERT(); MLD_LOCK_ASSERT(); KASSERT(mli->mli_version == MLD_VERSION_2, @@ -2998,7 +3001,7 @@ mld_v2_dispatch_general_query(struct mld_ifsoftc *mli) ifp = mli->mli_ifp; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET6 || ifma->ifma_protospec == NULL) continue; diff --git a/freebsd/sys/netinet6/mld6_var.h b/freebsd/sys/netinet6/mld6_var.h index 0aeac367..166c2055 100644 --- a/freebsd/sys/netinet6/mld6_var.h +++ b/freebsd/sys/netinet6/mld6_var.h @@ -136,7 +136,6 @@ struct mld_ifsoftc { uint32_t mli_qi; /* MLDv2 Query Interval (s) */ uint32_t mli_qri; /* MLDv2 Query Response Interval (s) */ uint32_t mli_uri; /* MLDv2 Unsolicited Report Interval (s) */ - SLIST_HEAD(,in6_multi) mli_relinmhead; /* released groups */ struct mbufq mli_gq; /* queue of general query responses */ }; diff --git a/freebsd/sys/netinet6/nd6.c b/freebsd/sys/netinet6/nd6.c index a00d5421..6a36803f 100644 --- a/freebsd/sys/netinet6/nd6.c +++ b/freebsd/sys/netinet6/nd6.c @@ -60,10 +60,8 @@ __FBSDID("$FreeBSD$"); #include <net/if.h> #include <net/if_var.h> -#include <net/if_arc.h> #include <net/if_dl.h> #include <net/if_types.h> -#include <net/fddi.h> #include <net/route.h> #include <net/vnet.h> @@ -305,7 +303,7 @@ nd6_ifdetach(struct ifnet *ifp, struct nd_ifinfo *nd) struct ifaddr *ifa, *next; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, next) { + CK_STAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, next) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; @@ -337,18 +335,7 @@ nd6_setmtu0(struct ifnet *ifp, struct nd_ifinfo *ndi) u_int32_t omaxmtu; omaxmtu = ndi->maxmtu; - - switch (ifp->if_type) { - case IFT_ARCNET: - ndi->maxmtu = MIN(ARC_PHDS_MAXMTU, ifp->if_mtu); /* RFC2497 */ - break; - case IFT_FDDI: - ndi->maxmtu = MIN(FDDIIPMTU, ifp->if_mtu); /* RFC2467 */ - break; - default: - ndi->maxmtu = ifp->if_mtu; - break; - } + ndi->maxmtu = ifp->if_mtu; /* * Decreasing the interface MTU under IPV6 minimum MTU may cause @@ -937,7 +924,7 @@ nd6_timer(void *arg) * XXXRW: in6_ifaddrhead locking. */ addrloop: - TAILQ_FOREACH_SAFE(ia6, &V_in6_ifaddrhead, ia_link, nia6) { + CK_STAILQ_FOREACH_SAFE(ia6, &V_in6_ifaddrhead, ia_link, nia6) { /* check address lifetime */ if (IFA6_IS_INVALID(ia6)) { int regen = 0; @@ -1083,7 +1070,7 @@ regen_tmpaddr(struct in6_ifaddr *ia6) ifp = ia6->ia_ifa.ifa_ifp; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { struct in6_ifaddr *it6; if (ifa->ifa_addr->sa_family != AF_INET6) @@ -1359,7 +1346,7 @@ restart: */ if (ifp->if_flags & IFF_POINTOPOINT) { IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != addr->sin6_family) continue; if (ifa->ifa_dstaddr != NULL && @@ -1702,7 +1689,7 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) * See RFC 4862, Section 5.4.5. */ IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; ia = (struct in6_ifaddr *)ifa; @@ -1732,7 +1719,7 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) if (V_ip6_dad_count > 0 && (ND_IFINFO(ifp)->flags & ND6_IFF_NO_DAD) == 0) { IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) @@ -1760,7 +1747,7 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) * assign one. */ IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) @@ -1804,7 +1791,7 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) while ((pr = LIST_FIRST(&prl)) != NULL) { LIST_REMOVE(pr, ndpr_entry); /* XXXRW: in6_ifaddrhead locking. */ - TAILQ_FOREACH_SAFE(ia, &V_in6_ifaddrhead, ia_link, + CK_STAILQ_FOREACH_SAFE(ia, &V_in6_ifaddrhead, ia_link, ia_next) { if ((ia->ia6_flags & IN6_IFF_AUTOCONF) == 0) continue; @@ -2148,7 +2135,7 @@ nd6_slowtimo(void *arg) callout_reset(&V_nd6_slowtimo_ch, ND6_SLOWTIMER_INTERVAL * hz, nd6_slowtimo, curvnet); IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (ifp->if_afdata[AF_INET6] == NULL) continue; nd6if = ND_IFINFO(ifp); @@ -2274,7 +2261,6 @@ nd6_resolve(struct ifnet *ifp, int is_gw, struct mbuf *m, if (m != NULL && m->m_flags & M_MCAST) { switch (ifp->if_type) { case IFT_ETHER: - case IFT_FDDI: case IFT_L2VLAN: case IFT_BRIDGE: ETHER_MAP_IPV6_MULTICAST(&dst6->sin6_addr, @@ -2526,15 +2512,13 @@ nd6_need_cache(struct ifnet *ifp) { /* * XXX: we currently do not make neighbor cache on any interface - * other than ARCnet, Ethernet, FDDI and GIF. + * other than Ethernet and GIF. * * RFC2893 says: * - unidirectional tunnels needs no ND */ switch (ifp->if_type) { - case IFT_ARCNET: case IFT_ETHER: - case IFT_FDDI: case IFT_IEEE1394: case IFT_L2VLAN: case IFT_INFINIBAND: diff --git a/freebsd/sys/netinet6/nd6_nbr.c b/freebsd/sys/netinet6/nd6_nbr.c index e5b37877..d4ab38af 100644 --- a/freebsd/sys/netinet6/nd6_nbr.c +++ b/freebsd/sys/netinet6/nd6_nbr.c @@ -1092,9 +1092,7 @@ caddr_t nd6_ifptomac(struct ifnet *ifp) { switch (ifp->if_type) { - case IFT_ARCNET: case IFT_ETHER: - case IFT_FDDI: case IFT_IEEE1394: case IFT_L2VLAN: case IFT_INFINIBAND: @@ -1468,7 +1466,6 @@ nd6_dad_duplicated(struct ifaddr *ifa, struct dadq *dp) */ switch (ifp->if_type) { case IFT_ETHER: - case IFT_FDDI: case IFT_ATM: case IFT_IEEE1394: case IFT_INFINIBAND: diff --git a/freebsd/sys/netinet6/nd6_rtr.c b/freebsd/sys/netinet6/nd6_rtr.c index 642faa1a..fab7c7c2 100644 --- a/freebsd/sys/netinet6/nd6_rtr.c +++ b/freebsd/sys/netinet6/nd6_rtr.c @@ -480,7 +480,7 @@ nd6_rtmsg(int cmd, struct rtentry *rt) ifp = rt->rt_ifp; if (ifp != NULL) { IF_ADDR_RLOCK(ifp); - ifa = TAILQ_FIRST(&ifp->if_addrhead); + ifa = CK_STAILQ_FIRST(&ifp->if_addrhead); info.rti_info[RTAX_IFP] = ifa->ifa_addr; ifa_ref(ifa); IF_ADDR_RUNLOCK(ifp); @@ -1349,7 +1349,7 @@ prelist_update(struct nd_prefixctl *new, struct nd_defrouter *dr, * "address". */ IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { struct in6_ifaddr *ifa6; u_int32_t remaininglifetime; @@ -1721,7 +1721,7 @@ restart: * The precise detection logic is same as the one for prefixes. */ IN6_IFADDR_RLOCK(&in6_ifa_tracker); - TAILQ_FOREACH(ifa, &V_in6_ifaddrhead, ia_link) { + CK_STAILQ_FOREACH(ifa, &V_in6_ifaddrhead, ia_link) { if (!(ifa->ia6_flags & IN6_IFF_AUTOCONF)) continue; @@ -1738,7 +1738,7 @@ restart: break; } if (ifa) { - TAILQ_FOREACH(ifa, &V_in6_ifaddrhead, ia_link) { + CK_STAILQ_FOREACH(ifa, &V_in6_ifaddrhead, ia_link) { if ((ifa->ia6_flags & IN6_IFF_AUTOCONF) == 0) continue; @@ -1756,7 +1756,7 @@ restart: } } } else { - TAILQ_FOREACH(ifa, &V_in6_ifaddrhead, ia_link) { + CK_STAILQ_FOREACH(ifa, &V_in6_ifaddrhead, ia_link) { if ((ifa->ia6_flags & IN6_IFF_AUTOCONF) == 0) continue; @@ -1910,7 +1910,7 @@ nd6_prefix_onlink(struct nd_prefix *pr) if (ifa == NULL) { /* XXX: freebsd does not have ifa_ifwithaf */ IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family == AF_INET6) { ifa_ref(ifa); break; diff --git a/freebsd/sys/netinet6/raw_ip6.c b/freebsd/sys/netinet6/raw_ip6.c index a4843380..c05399b3 100644 --- a/freebsd/sys/netinet6/raw_ip6.c +++ b/freebsd/sys/netinet6/raw_ip6.c @@ -736,23 +736,25 @@ rip6_bind(struct socket *so, struct sockaddr *nam, struct thread *td) return (EINVAL); if ((error = prison_check_ip6(td->td_ucred, &addr->sin6_addr)) != 0) return (error); - if (TAILQ_EMPTY(&V_ifnet) || addr->sin6_family != AF_INET6) + if (CK_STAILQ_EMPTY(&V_ifnet) || addr->sin6_family != AF_INET6) return (EADDRNOTAVAIL); if ((error = sa6_embedscope(addr, V_ip6_use_defzone)) != 0) return (error); + NET_EPOCH_ENTER(); if (!IN6_IS_ADDR_UNSPECIFIED(&addr->sin6_addr) && - (ifa = ifa_ifwithaddr((struct sockaddr *)addr)) == NULL) + (ifa = ifa_ifwithaddr((struct sockaddr *)addr)) == NULL) { + NET_EPOCH_EXIT(); return (EADDRNOTAVAIL); + } if (ifa != NULL && ((struct in6_ifaddr *)ifa)->ia6_flags & (IN6_IFF_ANYCAST|IN6_IFF_NOTREADY| IN6_IFF_DETACHED|IN6_IFF_DEPRECATED)) { - ifa_free(ifa); + NET_EPOCH_EXIT(); return (EADDRNOTAVAIL); } - if (ifa != NULL) - ifa_free(ifa); + NET_EPOCH_EXIT(); INP_INFO_WLOCK(&V_ripcbinfo); INP_WLOCK(inp); inp->in6p_laddr = addr->sin6_addr; @@ -774,7 +776,7 @@ rip6_connect(struct socket *so, struct sockaddr *nam, struct thread *td) if (nam->sa_len != sizeof(*addr)) return (EINVAL); - if (TAILQ_EMPTY(&V_ifnet)) + if (CK_STAILQ_EMPTY(&V_ifnet)) return (EADDRNOTAVAIL); if (addr->sin6_family != AF_INET6) return (EAFNOSUPPORT); diff --git a/freebsd/sys/netinet6/sctp6_usrreq.c b/freebsd/sys/netinet6/sctp6_usrreq.c index a79e6f53..fd963fb3 100644 --- a/freebsd/sys/netinet6/sctp6_usrreq.c +++ b/freebsd/sys/netinet6/sctp6_usrreq.c @@ -225,7 +225,7 @@ sctp6_notify(struct sctp_inpcb *inp, } break; case ICMP6_PACKET_TOO_BIG: - if ((net->dest_state & SCTP_ADDR_NO_PMTUD) == 0) { + if (net->dest_state & SCTP_ADDR_NO_PMTUD) { SCTP_TCB_UNLOCK(stcb); break; } diff --git a/freebsd/sys/netipsec/ipsec.h b/freebsd/sys/netipsec/ipsec.h index ac9361a8..936e7bca 100644 --- a/freebsd/sys/netipsec/ipsec.h +++ b/freebsd/sys/netipsec/ipsec.h @@ -219,8 +219,9 @@ struct ipsecstat { uint64_t ips_out_inval; /* output: generic error */ uint64_t ips_out_bundlesa; /* output: bundled SA processed */ - uint64_t ips_mbcoalesced; /* mbufs coalesced during clone */ - uint64_t ips_clcoalesced; /* clusters coalesced during clone */ + uint64_t ips_spdcache_hits; /* SPD cache hits */ + uint64_t ips_spdcache_misses; /* SPD cache misses */ + uint64_t ips_clcopied; /* clusters copied during clone */ uint64_t ips_mbinserted; /* mbufs inserted during makespace */ /* diff --git a/freebsd/sys/netipsec/ipsec_mbuf.c b/freebsd/sys/netipsec/ipsec_mbuf.c index 66d53514..f8e73c9e 100644 --- a/freebsd/sys/netipsec/ipsec_mbuf.c +++ b/freebsd/sys/netipsec/ipsec_mbuf.c @@ -257,10 +257,11 @@ m_striphdr(struct mbuf *m, int skip, int hlen) /* The header was at the beginning of the mbuf */ IPSECSTAT_INC(ips_input_front); m_adj(m1, hlen); - if ((m1->m_flags & M_PKTHDR) == 0) + if (m1 != m) m->m_pkthdr.len -= hlen; } else if (roff + hlen >= m1->m_len) { struct mbuf *mo; + int adjlen; /* * Part or all of the header is at the end of this mbuf, @@ -269,11 +270,13 @@ m_striphdr(struct mbuf *m, int skip, int hlen) */ IPSECSTAT_INC(ips_input_end); if (roff + hlen > m1->m_len) { + adjlen = roff + hlen - m1->m_len; + /* Adjust the next mbuf by the remainder */ - m_adj(m1->m_next, roff + hlen - m1->m_len); + m_adj(m1->m_next, adjlen); /* The second mbuf is guaranteed not to have a pkthdr... */ - m->m_pkthdr.len -= (roff + hlen - m1->m_len); + m->m_pkthdr.len -= adjlen; } /* Now, let's unlink the mbuf chain for a second...*/ @@ -281,9 +284,10 @@ m_striphdr(struct mbuf *m, int skip, int hlen) m1->m_next = NULL; /* ...and trim the end of the first part of the chain...sick */ - m_adj(m1, -(m1->m_len - roff)); - if ((m1->m_flags & M_PKTHDR) == 0) - m->m_pkthdr.len -= (m1->m_len - roff); + adjlen = m1->m_len - roff; + m_adj(m1, -adjlen); + if (m1 != m) + m->m_pkthdr.len -= adjlen; /* Finally, let's relink */ m1->m_next = mo; diff --git a/freebsd/sys/netipsec/key.c b/freebsd/sys/netipsec/key.c index 31269058..fbf12f41 100644 --- a/freebsd/sys/netipsec/key.c +++ b/freebsd/sys/netipsec/key.c @@ -175,6 +175,48 @@ static VNET_DEFINE(u_long, sphash_mask); #define SPHASH_HASHVAL(id) (key_u32hash(id) & V_sphash_mask) #define SPHASH_HASH(id) &V_sphashtbl[SPHASH_HASHVAL(id)] +/* SPD cache */ +struct spdcache_entry { + struct secpolicyindex spidx; /* secpolicyindex */ + struct secpolicy *sp; /* cached policy to be used */ + + LIST_ENTRY(spdcache_entry) chain; +}; +LIST_HEAD(spdcache_entry_list, spdcache_entry); + +#define SPDCACHE_MAX_ENTRIES_PER_HASH 8 + +static VNET_DEFINE(u_int, key_spdcache_maxentries) = 0; +#define V_key_spdcache_maxentries VNET(key_spdcache_maxentries) +static VNET_DEFINE(u_int, key_spdcache_threshold) = 32; +#define V_key_spdcache_threshold VNET(key_spdcache_threshold) +static VNET_DEFINE(unsigned long, spd_size) = 0; +#define V_spd_size VNET(spd_size) + +#define SPDCACHE_ENABLED() (V_key_spdcache_maxentries != 0) +#define SPDCACHE_ACTIVE() \ + (SPDCACHE_ENABLED() && V_spd_size >= V_key_spdcache_threshold) + +static VNET_DEFINE(struct spdcache_entry_list *, spdcachehashtbl); +static VNET_DEFINE(u_long, spdcachehash_mask); +#define V_spdcachehashtbl VNET(spdcachehashtbl) +#define V_spdcachehash_mask VNET(spdcachehash_mask) + +#define SPDCACHE_HASHVAL(idx) \ + (key_addrprotohash(&(idx)->src, &(idx)->dst, &(idx)->ul_proto) & \ + V_spdcachehash_mask) + +/* Each cache line is protected by a mutex */ +static VNET_DEFINE(struct mtx *, spdcache_lock); +#define V_spdcache_lock VNET(spdcache_lock) + +#define SPDCACHE_LOCK_INIT(a) \ + mtx_init(&V_spdcache_lock[a], "spdcache", \ + "fast ipsec SPD cache", MTX_DEF|MTX_DUPOK) +#define SPDCACHE_LOCK_DESTROY(a) mtx_destroy(&V_spdcache_lock[a]) +#define SPDCACHE_LOCK(a) mtx_lock(&V_spdcache_lock[a]); +#define SPDCACHE_UNLOCK(a) mtx_unlock(&V_spdcache_lock[a]); + /* SAD */ TAILQ_HEAD(secashead_queue, secashead); LIST_HEAD(secashead_list, secashead); @@ -200,8 +242,9 @@ static VNET_DEFINE(u_long, sahaddrhash_mask); #define SAHHASH_NHASH_LOG2 7 #define SAHHASH_NHASH (1 << SAHHASH_NHASH_LOG2) -#define SAHADDRHASH_HASHVAL(saidx) \ - (key_saidxhash(saidx) & V_sahaddrhash_mask) +#define SAHADDRHASH_HASHVAL(idx) \ + (key_addrprotohash(&(idx)->src, &(idx)->dst, &(idx)->proto) & \ + V_sahaddrhash_mask) #define SAHADDRHASH_HASH(saidx) \ &V_sahaddrhashtbl[SAHADDRHASH_HASHVAL(saidx)] @@ -217,33 +260,34 @@ static VNET_DEFINE(u_long, savhash_mask); #define SAVHASH_HASH(spi) &V_savhashtbl[SAVHASH_HASHVAL(spi)] static uint32_t -key_saidxhash(const struct secasindex *saidx) +key_addrprotohash(const union sockaddr_union *src, + const union sockaddr_union *dst, const uint8_t *proto) { uint32_t hval; - hval = fnv_32_buf(&saidx->proto, sizeof(saidx->proto), + hval = fnv_32_buf(proto, sizeof(*proto), FNV1_32_INIT); - switch (saidx->dst.sa.sa_family) { + switch (dst->sa.sa_family) { #ifdef INET case AF_INET: - hval = fnv_32_buf(&saidx->src.sin.sin_addr, + hval = fnv_32_buf(&src->sin.sin_addr, sizeof(in_addr_t), hval); - hval = fnv_32_buf(&saidx->dst.sin.sin_addr, + hval = fnv_32_buf(&dst->sin.sin_addr, sizeof(in_addr_t), hval); break; #endif #ifdef INET6 case AF_INET6: - hval = fnv_32_buf(&saidx->src.sin6.sin6_addr, + hval = fnv_32_buf(&src->sin6.sin6_addr, sizeof(struct in6_addr), hval); - hval = fnv_32_buf(&saidx->dst.sin6.sin6_addr, + hval = fnv_32_buf(&dst->sin6.sin6_addr, sizeof(struct in6_addr), hval); break; #endif default: hval = 0; ipseclog((LOG_DEBUG, "%s: unknown address family %d", - __func__, saidx->dst.sa.sa_family)); + __func__, dst->sa.sa_family)); } return (hval); } @@ -292,8 +336,9 @@ static VNET_DEFINE(u_long, acqseqhash_mask); #define ACQHASH_NHASH_LOG2 7 #define ACQHASH_NHASH (1 << ACQHASH_NHASH_LOG2) -#define ACQADDRHASH_HASHVAL(saidx) \ - (key_saidxhash(saidx) & V_acqaddrhash_mask) +#define ACQADDRHASH_HASHVAL(idx) \ + (key_addrprotohash(&(idx)->src, &(idx)->dst, &(idx)->proto) & \ + V_acqaddrhash_mask) #define ACQSEQHASH_HASHVAL(seq) \ (key_u32hash(seq) & V_acqseqhash_mask) #define ACQADDRHASH_HASH(saidx) \ @@ -465,6 +510,17 @@ SYSCTL_INT(_net_key, KEYCTL_AH_KEYMIN, ah_keymin, SYSCTL_INT(_net_key, KEYCTL_PREFERED_OLDSA, preferred_oldsa, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(key_preferred_oldsa), 0, ""); +static SYSCTL_NODE(_net_key, OID_AUTO, spdcache, CTLFLAG_RW, 0, "SPD cache"); + +SYSCTL_UINT(_net_key_spdcache, OID_AUTO, maxentries, + CTLFLAG_VNET | CTLFLAG_RDTUN, &VNET_NAME(key_spdcache_maxentries), 0, + "Maximum number of entries in the SPD cache" + " (power of 2, 0 to disable)"); + +SYSCTL_UINT(_net_key_spdcache, OID_AUTO, threshold, + CTLFLAG_VNET | CTLFLAG_RDTUN, &VNET_NAME(key_spdcache_threshold), 0, + "Number of SPs that make the SPD cache active"); + #define __LIST_CHAINED(elm) \ (!((elm)->chain.le_next == NULL && (elm)->chain.le_prev == NULL)) @@ -475,6 +531,7 @@ MALLOC_DEFINE(M_IPSEC_SR, "ipsecrequest", "ipsec security request"); MALLOC_DEFINE(M_IPSEC_MISC, "ipsec-misc", "ipsec miscellaneous"); MALLOC_DEFINE(M_IPSEC_SAQ, "ipsec-saq", "ipsec sa acquire"); MALLOC_DEFINE(M_IPSEC_SAR, "ipsec-reg", "ipsec sa acquire"); +MALLOC_DEFINE(M_IPSEC_SPDCACHE, "ipsec-spdcache", "ipsec SPD cache"); static VNET_DEFINE(uma_zone_t, key_lft_zone); #define V_key_lft_zone VNET(key_lft_zone) @@ -576,6 +633,7 @@ static struct callout key_timer; #endif static void key_unlink(struct secpolicy *); +static struct secpolicy *key_do_allocsp(struct secpolicyindex *spidx, u_int dir); static struct secpolicy *key_getsp(struct secpolicyindex *); static struct secpolicy *key_getspbyid(u_int32_t); static struct mbuf *key_gather_mbuf(struct mbuf *, @@ -696,6 +754,16 @@ static struct mbuf *key_setlifetime(struct seclifetime *, uint16_t); static struct mbuf *key_setkey(struct seckey *, uint16_t); static int xform_init(struct secasvar *, u_short); +static void spdcache_init(void); +static void spdcache_clear(void); +static struct spdcache_entry *spdcache_entry_alloc( + const struct secpolicyindex *spidx, + struct secpolicy *policy); +static void spdcache_entry_free(struct spdcache_entry *entry); +#ifdef VIMAGE +static void spdcache_destroy(void); +#endif + #define DBG_IPSEC_INITREF(t, p) do { \ refcount_init(&(p)->refcnt, 1); \ KEYDBG(KEY_STAMP, \ @@ -801,14 +869,8 @@ key_checksockaddrs(struct sockaddr *src, struct sockaddr *dst) return (0); } -/* - * allocating a SP for OUTBOUND or INBOUND packet. - * Must call key_freesp() later. - * OUT: NULL: not found - * others: found and return the pointer. - */ struct secpolicy * -key_allocsp(struct secpolicyindex *spidx, u_int dir) +key_do_allocsp(struct secpolicyindex *spidx, u_int dir) { SPTREE_RLOCK_TRACKER; struct secpolicy *sp; @@ -825,7 +887,73 @@ key_allocsp(struct secpolicyindex *spidx, u_int dir) } } SPTREE_RUNLOCK(); + return (sp); +} + + +/* + * allocating a SP for OUTBOUND or INBOUND packet. + * Must call key_freesp() later. + * OUT: NULL: not found + * others: found and return the pointer. + */ +struct secpolicy * +key_allocsp(struct secpolicyindex *spidx, u_int dir) +{ + struct spdcache_entry *entry, *lastentry, *tmpentry; + struct secpolicy *sp; + uint32_t hashv; + int nb_entries; + + if (!SPDCACHE_ACTIVE()) { + sp = key_do_allocsp(spidx, dir); + goto out; + } + + hashv = SPDCACHE_HASHVAL(spidx); + SPDCACHE_LOCK(hashv); + nb_entries = 0; + LIST_FOREACH_SAFE(entry, &V_spdcachehashtbl[hashv], chain, tmpentry) { + /* Removed outdated entries */ + if (entry->sp != NULL && + entry->sp->state == IPSEC_SPSTATE_DEAD) { + LIST_REMOVE(entry, chain); + spdcache_entry_free(entry); + continue; + } + + nb_entries++; + if (!key_cmpspidx_exactly(&entry->spidx, spidx)) { + lastentry = entry; + continue; + } + + sp = entry->sp; + if (entry->sp != NULL) + SP_ADDREF(sp); + + /* IPSECSTAT_INC(ips_spdcache_hits); */ + + SPDCACHE_UNLOCK(hashv); + goto out; + } + + /* IPSECSTAT_INC(ips_spdcache_misses); */ + sp = key_do_allocsp(spidx, dir); + entry = spdcache_entry_alloc(spidx, sp); + if (entry != NULL) { + if (nb_entries >= SPDCACHE_MAX_ENTRIES_PER_HASH) { + LIST_REMOVE(lastentry, chain); + spdcache_entry_free(lastentry); + } + + LIST_INSERT_HEAD(&V_spdcachehashtbl[hashv], entry, chain); + } + + SPDCACHE_UNLOCK(hashv); + +out: if (sp != NULL) { /* found a SPD entry */ sp->lastused = time_second; KEYDBG(IPSEC_STAMP, @@ -1109,9 +1237,12 @@ key_unlink(struct secpolicy *sp) } sp->state = IPSEC_SPSTATE_DEAD; TAILQ_REMOVE(&V_sptree[sp->spidx.dir], sp, chain); + V_spd_size--; LIST_REMOVE(sp, idhash); V_sp_genid++; SPTREE_WUNLOCK(); + if (SPDCACHE_ENABLED()) + spdcache_clear(); key_freesp(&sp); } @@ -1134,6 +1265,7 @@ key_insertsp(struct secpolicy *newsp) done: LIST_INSERT_HEAD(SPHASH_HASH(newsp->id), newsp, idhash); newsp->state = IPSEC_SPSTATE_ALIVE; + V_spd_size++; V_sp_genid++; } @@ -1209,9 +1341,12 @@ key_unregister_ifnet(struct secpolicy **spp, u_int count) spp[i]->state = IPSEC_SPSTATE_DEAD; TAILQ_REMOVE(&V_sptree_ifnet[spp[i]->spidx.dir], spp[i], chain); + V_spd_size--; LIST_REMOVE(spp[i], idhash); } SPTREE_WUNLOCK(); + if (SPDCACHE_ENABLED()) + spdcache_clear(); for (i = 0; i < count; i++) { m = key_setdumpsp(spp[i], SADB_X_SPDDELETE, 0, 0); @@ -1941,6 +2076,8 @@ key_spdadd(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp) } key_insertsp(newsp); SPTREE_WUNLOCK(); + if (SPDCACHE_ENABLED()) + spdcache_clear(); KEYDBG(KEY_STAMP, printf("%s: SP(%p)\n", __func__, newsp)); @@ -2395,7 +2532,10 @@ key_spdflush(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp) LIST_REMOVE(sp, idhash); } V_sp_genid++; + V_spd_size = 0; SPTREE_WUNLOCK(); + if (SPDCACHE_ENABLED()) + spdcache_clear(); sp = TAILQ_FIRST(&drainq); while (sp != NULL) { nextsp = TAILQ_NEXT(sp, chain); @@ -4072,7 +4212,8 @@ key_cmpspidx_exactly(struct secpolicyindex *spidx0, if (spidx0->prefs != spidx1->prefs || spidx0->prefd != spidx1->prefd - || spidx0->ul_proto != spidx1->ul_proto) + || spidx0->ul_proto != spidx1->ul_proto + || spidx0->dir != spidx1->dir) return 0; return key_sockaddrcmp(&spidx0->src.sa, &spidx1->src.sa, 1) == 0 && @@ -4340,12 +4481,15 @@ key_flush_spd(time_t now) continue; } TAILQ_REMOVE(&V_sptree[sp->spidx.dir], sp, chain); + V_spd_size--; LIST_REMOVE(sp, idhash); sp->state = IPSEC_SPSTATE_DEAD; sp = nextsp; } V_sp_genid++; SPTREE_WUNLOCK(); + if (SPDCACHE_ENABLED()) + spdcache_clear(); sp = LIST_FIRST(&drainq); while (sp != NULL) { @@ -8069,6 +8213,96 @@ key_validate_ext(const struct sadb_ext *ext, int len) } void +spdcache_init(void) +{ + int i; + + TUNABLE_INT_FETCH("net.key.spdcache.maxentries", + &V_key_spdcache_maxentries); + TUNABLE_INT_FETCH("net.key.spdcache.threshold", + &V_key_spdcache_threshold); + + if (V_key_spdcache_maxentries) { + V_key_spdcache_maxentries = MAX(V_key_spdcache_maxentries, + SPDCACHE_MAX_ENTRIES_PER_HASH); + V_spdcachehashtbl = hashinit(V_key_spdcache_maxentries / + SPDCACHE_MAX_ENTRIES_PER_HASH, + M_IPSEC_SPDCACHE, &V_spdcachehash_mask); + V_key_spdcache_maxentries = (V_spdcachehash_mask + 1) + * SPDCACHE_MAX_ENTRIES_PER_HASH; + + V_spdcache_lock = malloc(sizeof(struct mtx) * + (V_spdcachehash_mask + 1), + M_IPSEC_SPDCACHE, M_WAITOK|M_ZERO); + + for (i = 0; i < V_spdcachehash_mask + 1; ++i) + SPDCACHE_LOCK_INIT(i); + } +} + +struct spdcache_entry * +spdcache_entry_alloc(const struct secpolicyindex *spidx, struct secpolicy *sp) +{ + struct spdcache_entry *entry; + + entry = malloc(sizeof(struct spdcache_entry), + M_IPSEC_SPDCACHE, M_NOWAIT|M_ZERO); + if (entry == NULL) + return NULL; + + if (sp != NULL) + SP_ADDREF(sp); + + entry->spidx = *spidx; + entry->sp = sp; + + return (entry); +} + +void +spdcache_entry_free(struct spdcache_entry *entry) +{ + + if (entry->sp != NULL) + key_freesp(&entry->sp); + free(entry, M_IPSEC_SPDCACHE); +} + +void +spdcache_clear(void) +{ + struct spdcache_entry *entry; + int i; + + for (i = 0; i < V_spdcachehash_mask + 1; ++i) { + SPDCACHE_LOCK(i); + while (!LIST_EMPTY(&V_spdcachehashtbl[i])) { + entry = LIST_FIRST(&V_spdcachehashtbl[i]); + LIST_REMOVE(entry, chain); + spdcache_entry_free(entry); + } + SPDCACHE_UNLOCK(i); + } +} + +#ifdef VIMAGE +void +spdcache_destroy(void) +{ + int i; + + if (SPDCACHE_ENABLED()) { + spdcache_clear(); + hashdestroy(V_spdcachehashtbl, M_IPSEC_SPDCACHE, V_spdcachehash_mask); + + for (i = 0; i < V_spdcachehash_mask + 1; ++i) + SPDCACHE_LOCK_DESTROY(i); + + free(V_spdcache_lock, M_IPSEC_SPDCACHE); + } +} +#endif +void key_init(void) { int i; @@ -8092,6 +8326,8 @@ key_init(void) V_acqseqhashtbl = hashinit(ACQHASH_NHASH, M_IPSEC_SAQ, &V_acqseqhash_mask); + spdcache_init(); + for (i = 0; i <= SADB_SATYPE_MAX; i++) LIST_INIT(&V_regtree[i]); @@ -8147,6 +8383,7 @@ key_destroy(void) for (i = 0; i < V_sphash_mask + 1; i++) LIST_INIT(&V_sphashtbl[i]); SPTREE_WUNLOCK(); + spdcache_destroy(); sp = TAILQ_FIRST(&drainq); while (sp != NULL) { diff --git a/freebsd/sys/netpfil/pf/if_pfsync.c b/freebsd/sys/netpfil/pf/if_pfsync.c index 3ed1b3bb..9b457818 100644 --- a/freebsd/sys/netpfil/pf/if_pfsync.c +++ b/freebsd/sys/netpfil/pf/if_pfsync.c @@ -589,6 +589,8 @@ pfsync_input(struct mbuf **mp, int *offp __unused, int proto __unused) int rv; uint16_t count; + PF_RULES_RLOCK_TRACKER; + *mp = NULL; V_pfsyncstats.pfsyncs_ipackets++; diff --git a/freebsd/sys/netpfil/pf/pf.c b/freebsd/sys/netpfil/pf/pf.c index 1aab6f49..3cc4ff11 100644 --- a/freebsd/sys/netpfil/pf/pf.c +++ b/freebsd/sys/netpfil/pf/pf.c @@ -371,11 +371,14 @@ u_long pf_hashmask; u_long pf_srchashmask; static u_long pf_hashsize; static u_long pf_srchashsize; +u_long pf_ioctl_maxcount = 65535; SYSCTL_ULONG(_net_pf, OID_AUTO, states_hashsize, CTLFLAG_RDTUN, &pf_hashsize, 0, "Size of pf(4) states hashtable"); SYSCTL_ULONG(_net_pf, OID_AUTO, source_nodes_hashsize, CTLFLAG_RDTUN, &pf_srchashsize, 0, "Size of pf(4) source nodes hashtable"); +SYSCTL_ULONG(_net_pf, OID_AUTO, request_maxcount, CTLFLAG_RDTUN, + &pf_ioctl_maxcount, 0, "Maximum number of tables, addresses, ... in a single ioctl() call"); VNET_DEFINE(void *, pf_swi_cookie); @@ -5736,8 +5739,7 @@ bad: /* * FreeBSD supports cksum offloads for the following drivers. - * em(4), fxp(4), ixgb(4), lge(4), ndis(4), nge(4), re(4), - * ti(4), txp(4), xl(4) + * em(4), fxp(4), lge(4), ndis(4), nge(4), re(4), ti(4), txp(4), xl(4) * * CSUM_DATA_VALID | CSUM_PSEUDO_HDR : * network driver performed cksum including pseudo header, need to verify @@ -5884,6 +5886,8 @@ pf_test(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0, struct inpcb * struct pf_pdesc pd; int off, dirndx, pqid = 0; + PF_RULES_RLOCK_TRACKER; + M_ASSERTPKTHDR(m); if (!V_pf_status.running) @@ -6271,6 +6275,7 @@ pf_test6(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0, struct inpcb struct pf_pdesc pd; int off, terminal = 0, dirndx, rh_cnt = 0, pqid = 0; + PF_RULES_RLOCK_TRACKER; M_ASSERTPKTHDR(m); if (!V_pf_status.running) diff --git a/freebsd/sys/netpfil/pf/pf_if.c b/freebsd/sys/netpfil/pf/pf_if.c index e224e6a7..2ac76ff2 100644 --- a/freebsd/sys/netpfil/pf/pf_if.c +++ b/freebsd/sys/netpfil/pf/pf_if.c @@ -48,7 +48,6 @@ __FBSDID("$FreeBSD$"); #include <sys/eventhandler.h> #include <sys/lock.h> #include <sys/mbuf.h> -#include <sys/rwlock.h> #include <sys/socket.h> #include <net/if.h> @@ -131,9 +130,9 @@ pfi_initialize_vnet(void) PF_RULES_WUNLOCK(); IFNET_RLOCK(); - TAILQ_FOREACH(ifg, &V_ifg_head, ifg_next) + CK_STAILQ_FOREACH(ifg, &V_ifg_head, ifg_next) pfi_attach_ifgroup(ifg); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) pfi_attach_ifnet(ifp); IFNET_RUNLOCK(); } @@ -302,7 +301,7 @@ pfi_kif_match(struct pfi_kif *rule_kif, struct pfi_kif *packet_kif) if (rule_kif->pfik_group != NULL) /* XXXGL: locking? */ - TAILQ_FOREACH(p, &packet_kif->pfik_ifp->if_groups, ifgl_next) + CK_STAILQ_FOREACH(p, &packet_kif->pfik_ifp->if_groups, ifgl_next) if (p->ifgl_group == rule_kif->pfik_group) return (1); @@ -468,7 +467,7 @@ pfi_kif_update(struct pfi_kif *kif) /* again for all groups kif is member of */ if (kif->pfik_ifp != NULL) { IF_ADDR_RLOCK(kif->pfik_ifp); - TAILQ_FOREACH(ifgl, &kif->pfik_ifp->if_groups, ifgl_next) + CK_STAILQ_FOREACH(ifgl, &kif->pfik_ifp->if_groups, ifgl_next) pfi_kif_update((struct pfi_kif *) ifgl->ifgl_group->ifg_pf_kif); IF_ADDR_RUNLOCK(kif->pfik_ifp); @@ -508,7 +507,7 @@ pfi_table_update(struct pfr_ktable *kt, struct pfi_kif *kif, int net, int flags) pfi_instance_add(kif->pfik_ifp, net, flags); else if (kif->pfik_group != NULL) { IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifgm, &kif->pfik_group->ifg_members, ifgm_next) + CK_STAILQ_FOREACH(ifgm, &kif->pfik_group->ifg_members, ifgm_next) pfi_instance_add(ifgm->ifgm_ifp, net, flags); IFNET_RUNLOCK_NOSLEEP(); } @@ -527,7 +526,7 @@ pfi_instance_add(struct ifnet *ifp, int net, int flags) int net2, af; IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ia, &ifp->if_addrhead, ifa_link) { + CK_STAILQ_FOREACH(ia, &ifp->if_addrhead, ifa_link) { if (ia->ifa_addr == NULL) continue; af = ia->ifa_addr->sa_family; @@ -670,7 +669,7 @@ pfi_update_status(const char *name, struct pf_status *pfs) struct pfi_kif *p; struct pfi_kif_cmp key; struct ifg_member p_member, *ifgm; - TAILQ_HEAD(, ifg_member) ifg_members; + CK_STAILQ_HEAD(, ifg_member) ifg_members; int i, j, k; strlcpy(key.pfik_name, name, sizeof(key.pfik_name)); @@ -685,14 +684,14 @@ pfi_update_status(const char *name, struct pf_status *pfs) /* build a temporary list for p only */ bzero(&p_member, sizeof(p_member)); p_member.ifgm_ifp = p->pfik_ifp; - TAILQ_INIT(&ifg_members); - TAILQ_INSERT_TAIL(&ifg_members, &p_member, ifgm_next); + CK_STAILQ_INIT(&ifg_members); + CK_STAILQ_INSERT_TAIL(&ifg_members, &p_member, ifgm_next); } if (pfs) { bzero(pfs->pcounters, sizeof(pfs->pcounters)); bzero(pfs->bcounters, sizeof(pfs->bcounters)); } - TAILQ_FOREACH(ifgm, &ifg_members, ifgm_next) { + CK_STAILQ_FOREACH(ifgm, &ifg_members, ifgm_next) { if (ifgm->ifgm_ifp == NULL || ifgm->ifgm_ifp->if_pf_kif == NULL) continue; p = (struct pfi_kif *)ifgm->ifgm_ifp->if_pf_kif; diff --git a/freebsd/sys/netpfil/pf/pf_ioctl.c b/freebsd/sys/netpfil/pf/pf_ioctl.c index 9be57273..837ad31c 100644 --- a/freebsd/sys/netpfil/pf/pf_ioctl.c +++ b/freebsd/sys/netpfil/pf/pf_ioctl.c @@ -61,7 +61,6 @@ __FBSDID("$FreeBSD$"); #include <sys/mbuf.h> #include <sys/module.h> #include <sys/proc.h> -#include <sys/rwlock.h> #include <sys/smp.h> #include <sys/socket.h> #include <sys/sysctl.h> @@ -204,7 +203,7 @@ VNET_DEFINE(int, pf_vnet_active); int pf_end_threads; struct proc *pf_purge_proc; -struct rwlock pf_rules_lock; +struct rmlock pf_rules_lock; struct sx pf_ioctl_lock; struct sx pf_end_lock; @@ -218,6 +217,8 @@ pfsync_defer_t *pfsync_defer_ptr = NULL; /* pflog */ pflog_packet_t *pflog_packet_ptr = NULL; +extern u_long pf_ioctl_maxcount; + static void pfattach_vnet(void) { @@ -995,6 +996,7 @@ static int pfioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, struct thread *td) { int error = 0; + PF_RULES_RLOCK_TRACKER; /* XXX keep in sync with switch() below */ if (securelevel_gt(td->td_ucred, 2)) @@ -2542,13 +2544,16 @@ DIOCCHANGEADDR_error: error = ENODEV; break; } - totlen = io->pfrio_size * sizeof(struct pfr_table); - pfrts = mallocarray(io->pfrio_size, sizeof(struct pfr_table), - M_TEMP, M_WAITOK); - if (! pfrts) { + + if (io->pfrio_size < 0 || io->pfrio_size > pf_ioctl_maxcount || + WOULD_OVERFLOW(io->pfrio_size, sizeof(struct pfr_table))) { error = ENOMEM; break; } + + totlen = io->pfrio_size * sizeof(struct pfr_table); + pfrts = mallocarray(io->pfrio_size, sizeof(struct pfr_table), + M_TEMP, M_WAITOK); error = copyin(io->pfrio_buffer, pfrts, totlen); if (error) { free(pfrts, M_TEMP); @@ -2571,13 +2576,16 @@ DIOCCHANGEADDR_error: error = ENODEV; break; } - totlen = io->pfrio_size * sizeof(struct pfr_table); - pfrts = mallocarray(io->pfrio_size, sizeof(struct pfr_table), - M_TEMP, M_WAITOK); - if (! pfrts) { + + if (io->pfrio_size < 0 || io->pfrio_size > pf_ioctl_maxcount || + WOULD_OVERFLOW(io->pfrio_size, sizeof(struct pfr_table))) { error = ENOMEM; break; } + + totlen = io->pfrio_size * sizeof(struct pfr_table); + pfrts = mallocarray(io->pfrio_size, sizeof(struct pfr_table), + M_TEMP, M_WAITOK); error = copyin(io->pfrio_buffer, pfrts, totlen); if (error) { free(pfrts, M_TEMP); @@ -2594,20 +2602,25 @@ DIOCCHANGEADDR_error: case DIOCRGETTABLES: { struct pfioc_table *io = (struct pfioc_table *)addr; struct pfr_table *pfrts; - size_t totlen; + size_t totlen, n; if (io->pfrio_esize != sizeof(struct pfr_table)) { error = ENODEV; break; } + PF_RULES_RLOCK(); + n = pfr_table_count(&io->pfrio_table, io->pfrio_flags); + io->pfrio_size = min(io->pfrio_size, n); + totlen = io->pfrio_size * sizeof(struct pfr_table); + pfrts = mallocarray(io->pfrio_size, sizeof(struct pfr_table), - M_TEMP, M_WAITOK); - if (! pfrts) { + M_TEMP, M_NOWAIT); + if (pfrts == NULL) { error = ENOMEM; + PF_RULES_RUNLOCK(); break; } - PF_RULES_RLOCK(); error = pfr_get_tables(&io->pfrio_table, pfrts, &io->pfrio_size, io->pfrio_flags | PFR_FLAG_USERIOCTL); PF_RULES_RUNLOCK(); @@ -2620,20 +2633,24 @@ DIOCCHANGEADDR_error: case DIOCRGETTSTATS: { struct pfioc_table *io = (struct pfioc_table *)addr; struct pfr_tstats *pfrtstats; - size_t totlen; + size_t totlen, n; if (io->pfrio_esize != sizeof(struct pfr_tstats)) { error = ENODEV; break; } + PF_RULES_WLOCK(); + n = pfr_table_count(&io->pfrio_table, io->pfrio_flags); + io->pfrio_size = min(io->pfrio_size, n); + totlen = io->pfrio_size * sizeof(struct pfr_tstats); pfrtstats = mallocarray(io->pfrio_size, - sizeof(struct pfr_tstats), M_TEMP, M_WAITOK); - if (! pfrtstats) { + sizeof(struct pfr_tstats), M_TEMP, M_NOWAIT); + if (pfrtstats == NULL) { error = ENOMEM; + PF_RULES_WUNLOCK(); break; } - PF_RULES_WLOCK(); error = pfr_get_tstats(&io->pfrio_table, pfrtstats, &io->pfrio_size, io->pfrio_flags | PFR_FLAG_USERIOCTL); PF_RULES_WUNLOCK(); @@ -2646,25 +2663,31 @@ DIOCCHANGEADDR_error: case DIOCRCLRTSTATS: { struct pfioc_table *io = (struct pfioc_table *)addr; struct pfr_table *pfrts; - size_t totlen; + size_t totlen, n; if (io->pfrio_esize != sizeof(struct pfr_table)) { error = ENODEV; break; } + + PF_RULES_WLOCK(); + n = pfr_table_count(&io->pfrio_table, io->pfrio_flags); + io->pfrio_size = min(io->pfrio_size, n); + totlen = io->pfrio_size * sizeof(struct pfr_table); pfrts = mallocarray(io->pfrio_size, sizeof(struct pfr_table), - M_TEMP, M_WAITOK); - if (! pfrts) { + M_TEMP, M_NOWAIT); + if (pfrts == NULL) { error = ENOMEM; + PF_RULES_WUNLOCK(); break; } error = copyin(io->pfrio_buffer, pfrts, totlen); if (error) { free(pfrts, M_TEMP); + PF_RULES_WUNLOCK(); break; } - PF_RULES_WLOCK(); error = pfr_clr_tstats(pfrts, io->pfrio_size, &io->pfrio_nzero, io->pfrio_flags | PFR_FLAG_USERIOCTL); PF_RULES_WUNLOCK(); @@ -2675,25 +2698,31 @@ DIOCCHANGEADDR_error: case DIOCRSETTFLAGS: { struct pfioc_table *io = (struct pfioc_table *)addr; struct pfr_table *pfrts; - size_t totlen; + size_t totlen, n; if (io->pfrio_esize != sizeof(struct pfr_table)) { error = ENODEV; break; } + + PF_RULES_WLOCK(); + n = pfr_table_count(&io->pfrio_table, io->pfrio_flags); + io->pfrio_size = min(io->pfrio_size, n); + totlen = io->pfrio_size * sizeof(struct pfr_table); pfrts = mallocarray(io->pfrio_size, sizeof(struct pfr_table), - M_TEMP, M_WAITOK); - if (! pfrts) { + M_TEMP, M_NOWAIT); + if (pfrts == NULL) { error = ENOMEM; + PF_RULES_WUNLOCK(); break; } error = copyin(io->pfrio_buffer, pfrts, totlen); if (error) { free(pfrts, M_TEMP); + PF_RULES_WUNLOCK(); break; } - PF_RULES_WLOCK(); error = pfr_set_tflags(pfrts, io->pfrio_size, io->pfrio_setflag, io->pfrio_clrflag, &io->pfrio_nchange, &io->pfrio_ndel, io->pfrio_flags | PFR_FLAG_USERIOCTL); @@ -2725,9 +2754,15 @@ DIOCCHANGEADDR_error: error = ENODEV; break; } + if (io->pfrio_size < 0 || + io->pfrio_size > pf_ioctl_maxcount || + WOULD_OVERFLOW(io->pfrio_size, sizeof(struct pfr_addr))) { + error = EINVAL; + break; + } totlen = io->pfrio_size * sizeof(struct pfr_addr); pfras = mallocarray(io->pfrio_size, sizeof(struct pfr_addr), - M_TEMP, M_WAITOK); + M_TEMP, M_NOWAIT); if (! pfras) { error = ENOMEM; break; @@ -2757,9 +2792,15 @@ DIOCCHANGEADDR_error: error = ENODEV; break; } + if (io->pfrio_size < 0 || + io->pfrio_size > pf_ioctl_maxcount || + WOULD_OVERFLOW(io->pfrio_size, sizeof(struct pfr_addr))) { + error = EINVAL; + break; + } totlen = io->pfrio_size * sizeof(struct pfr_addr); pfras = mallocarray(io->pfrio_size, sizeof(struct pfr_addr), - M_TEMP, M_WAITOK); + M_TEMP, M_NOWAIT); if (! pfras) { error = ENOMEM; break; @@ -2789,10 +2830,19 @@ DIOCCHANGEADDR_error: error = ENODEV; break; } + if (io->pfrio_size < 0 || io->pfrio_size2 < 0) { + error = EINVAL; + break; + } count = max(io->pfrio_size, io->pfrio_size2); + if (count > pf_ioctl_maxcount || + WOULD_OVERFLOW(count, sizeof(struct pfr_addr))) { + error = EINVAL; + break; + } totlen = count * sizeof(struct pfr_addr); pfras = mallocarray(count, sizeof(struct pfr_addr), M_TEMP, - M_WAITOK); + M_NOWAIT); if (! pfras) { error = ENOMEM; break; @@ -2823,9 +2873,15 @@ DIOCCHANGEADDR_error: error = ENODEV; break; } + if (io->pfrio_size < 0 || + io->pfrio_size > pf_ioctl_maxcount || + WOULD_OVERFLOW(io->pfrio_size, sizeof(struct pfr_addr))) { + error = EINVAL; + break; + } totlen = io->pfrio_size * sizeof(struct pfr_addr); pfras = mallocarray(io->pfrio_size, sizeof(struct pfr_addr), - M_TEMP, M_WAITOK); + M_TEMP, M_NOWAIT); if (! pfras) { error = ENOMEM; break; @@ -2849,9 +2905,15 @@ DIOCCHANGEADDR_error: error = ENODEV; break; } + if (io->pfrio_size < 0 || + io->pfrio_size > pf_ioctl_maxcount || + WOULD_OVERFLOW(io->pfrio_size, sizeof(struct pfr_astats))) { + error = EINVAL; + break; + } totlen = io->pfrio_size * sizeof(struct pfr_astats); pfrastats = mallocarray(io->pfrio_size, - sizeof(struct pfr_astats), M_TEMP, M_WAITOK); + sizeof(struct pfr_astats), M_TEMP, M_NOWAIT); if (! pfrastats) { error = ENOMEM; break; @@ -2875,9 +2937,15 @@ DIOCCHANGEADDR_error: error = ENODEV; break; } + if (io->pfrio_size < 0 || + io->pfrio_size > pf_ioctl_maxcount || + WOULD_OVERFLOW(io->pfrio_size, sizeof(struct pfr_addr))) { + error = EINVAL; + break; + } totlen = io->pfrio_size * sizeof(struct pfr_addr); pfras = mallocarray(io->pfrio_size, sizeof(struct pfr_addr), - M_TEMP, M_WAITOK); + M_TEMP, M_NOWAIT); if (! pfras) { error = ENOMEM; break; @@ -2907,9 +2975,15 @@ DIOCCHANGEADDR_error: error = ENODEV; break; } + if (io->pfrio_size < 0 || + io->pfrio_size > pf_ioctl_maxcount || + WOULD_OVERFLOW(io->pfrio_size, sizeof(struct pfr_addr))) { + error = EINVAL; + break; + } totlen = io->pfrio_size * sizeof(struct pfr_addr); pfras = mallocarray(io->pfrio_size, sizeof(struct pfr_addr), - M_TEMP, M_WAITOK); + M_TEMP, M_NOWAIT); if (! pfras) { error = ENOMEM; break; @@ -2939,9 +3013,15 @@ DIOCCHANGEADDR_error: error = ENODEV; break; } + if (io->pfrio_size < 0 || + io->pfrio_size > pf_ioctl_maxcount || + WOULD_OVERFLOW(io->pfrio_size, sizeof(struct pfr_addr))) { + error = EINVAL; + break; + } totlen = io->pfrio_size * sizeof(struct pfr_addr); pfras = mallocarray(io->pfrio_size, sizeof(struct pfr_addr), - M_TEMP, M_WAITOK); + M_TEMP, M_NOWAIT); if (! pfras) { error = ENOMEM; break; @@ -2986,9 +3066,15 @@ DIOCCHANGEADDR_error: error = ENODEV; break; } + if (io->size < 0 || + io->size > pf_ioctl_maxcount || + WOULD_OVERFLOW(io->size, sizeof(struct pfioc_trans_e))) { + error = EINVAL; + break; + } totlen = sizeof(struct pfioc_trans_e) * io->size; ioes = mallocarray(io->size, sizeof(struct pfioc_trans_e), - M_TEMP, M_WAITOK); + M_TEMP, M_NOWAIT); if (! ioes) { error = ENOMEM; break; @@ -3057,9 +3143,15 @@ DIOCCHANGEADDR_error: error = ENODEV; break; } + if (io->size < 0 || + io->size > pf_ioctl_maxcount || + WOULD_OVERFLOW(io->size, sizeof(struct pfioc_trans_e))) { + error = EINVAL; + break; + } totlen = sizeof(struct pfioc_trans_e) * io->size; ioes = mallocarray(io->size, sizeof(struct pfioc_trans_e), - M_TEMP, M_WAITOK); + M_TEMP, M_NOWAIT); if (! ioes) { error = ENOMEM; break; @@ -3128,10 +3220,18 @@ DIOCCHANGEADDR_error: error = ENODEV; break; } + + if (io->size < 0 || + io->size > pf_ioctl_maxcount || + WOULD_OVERFLOW(io->size, sizeof(struct pfioc_trans_e))) { + error = EINVAL; + break; + } + totlen = sizeof(struct pfioc_trans_e) * io->size; ioes = mallocarray(io->size, sizeof(struct pfioc_trans_e), - M_TEMP, M_WAITOK); - if (! ioes) { + M_TEMP, M_NOWAIT); + if (ioes == NULL) { error = ENOMEM; break; } @@ -3334,13 +3434,21 @@ DIOCCHANGEADDR_error: break; } + if (io->pfiio_size < 0 || + io->pfiio_size > pf_ioctl_maxcount || + WOULD_OVERFLOW(io->pfiio_size, sizeof(struct pfi_kif))) { + error = EINVAL; + break; + } + bufsiz = io->pfiio_size * sizeof(struct pfi_kif); ifstore = mallocarray(io->pfiio_size, sizeof(struct pfi_kif), - M_TEMP, M_WAITOK); - if (! ifstore) { + M_TEMP, M_NOWAIT); + if (ifstore == NULL) { error = ENOMEM; break; } + PF_RULES_RLOCK(); pfi_get_ifaces(io->pfiio_name, ifstore, &io->pfiio_size); PF_RULES_RUNLOCK(); @@ -3828,7 +3936,7 @@ pf_load(void) { int error; - rw_init(&pf_rules_lock, "pf rulesets"); + rm_init(&pf_rules_lock, "pf rulesets"); sx_init(&pf_ioctl_lock, "pf ioctl"); sx_init(&pf_end_lock, "pf end thread"); @@ -3901,7 +4009,7 @@ pf_unload(void) pfi_cleanup(); - rw_destroy(&pf_rules_lock); + rm_destroy(&pf_rules_lock); sx_destroy(&pf_ioctl_lock); sx_destroy(&pf_end_lock); } diff --git a/freebsd/sys/netpfil/pf/pf_lb.c b/freebsd/sys/netpfil/pf/pf_lb.c index 7ce27a07..1fd02a0b 100644 --- a/freebsd/sys/netpfil/pf/pf_lb.c +++ b/freebsd/sys/netpfil/pf/pf_lb.c @@ -48,7 +48,6 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/lock.h> #include <sys/mbuf.h> -#include <sys/rwlock.h> #include <sys/socket.h> #include <sys/sysctl.h> diff --git a/freebsd/sys/netpfil/pf/pf_norm.c b/freebsd/sys/netpfil/pf/pf_norm.c index 4f0966ed..61da5e4f 100644 --- a/freebsd/sys/netpfil/pf/pf_norm.c +++ b/freebsd/sys/netpfil/pf/pf_norm.c @@ -43,7 +43,6 @@ __FBSDID("$FreeBSD$"); #include <sys/mbuf.h> #include <sys/mutex.h> #include <sys/refcount.h> -#include <sys/rwlock.h> #include <sys/socket.h> #include <net/if.h> diff --git a/freebsd/sys/netpfil/pf/pf_osfp.c b/freebsd/sys/netpfil/pf/pf_osfp.c index 1ee16df5..b87d39bd 100644 --- a/freebsd/sys/netpfil/pf/pf_osfp.c +++ b/freebsd/sys/netpfil/pf/pf_osfp.c @@ -29,7 +29,6 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/lock.h> #include <sys/mbuf.h> -#include <sys/rwlock.h> #include <sys/socket.h> #include <netinet/in.h> diff --git a/freebsd/sys/netpfil/pf/pf_table.c b/freebsd/sys/netpfil/pf/pf_table.c index 06916204..04a275d9 100644 --- a/freebsd/sys/netpfil/pf/pf_table.c +++ b/freebsd/sys/netpfil/pf/pf_table.c @@ -46,7 +46,6 @@ __FBSDID("$FreeBSD$"); #include <sys/mbuf.h> #include <sys/mutex.h> #include <sys/refcount.h> -#include <sys/rwlock.h> #include <sys/socket.h> #include <vm/uma.h> @@ -179,7 +178,6 @@ static struct pfr_ktable *pfr_lookup_table(struct pfr_table *); static void pfr_clean_node_mask(struct pfr_ktable *, struct pfr_kentryworkq *); -static int pfr_table_count(struct pfr_table *, int); static int pfr_skip_table(struct pfr_table *, struct pfr_ktable *, int); static struct pfr_kentry @@ -1690,7 +1688,7 @@ pfr_fix_anchor(char *anchor) return (0); } -static int +int pfr_table_count(struct pfr_table *filter, int flags) { struct pf_ruleset *rs; diff --git a/freebsd/sys/opencrypto/cryptodev.c b/freebsd/sys/opencrypto/cryptodev.c index 3a9ed6e3..162a247c 100644 --- a/freebsd/sys/opencrypto/cryptodev.c +++ b/freebsd/sys/opencrypto/cryptodev.c @@ -43,8 +43,6 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); -#include <rtems/bsd/local/opt_compat.h> - #include <sys/param.h> #include <sys/systm.h> #include <sys/malloc.h> diff --git a/freebsd/sys/opencrypto/cryptosoft.c b/freebsd/sys/opencrypto/cryptosoft.c index 55e98b13..69993ae0 100644 --- a/freebsd/sys/opencrypto/cryptosoft.c +++ b/freebsd/sys/opencrypto/cryptosoft.c @@ -86,7 +86,7 @@ static int swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, int flags) { - unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN], *idat; + unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN]; unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN]; struct enc_xform *exf; int i, j, k, blks, ind, count, ivlen; @@ -251,11 +251,13 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, } while (uio->uio_iov[ind].iov_len >= k + blks && i > 0) { + uint8_t *idat; size_t nb, rem; nb = blks; - rem = uio->uio_iov[ind].iov_len - k; - idat = (char *)uio->uio_iov[ind].iov_base + k; + rem = MIN((size_t)i, + uio->uio_iov[ind].iov_len - (size_t)k); + idat = (uint8_t *)uio->uio_iov[ind].iov_base + k; if (exf->reinit) { if ((crd->crd_flags & CRD_F_ENCRYPT) != 0 && @@ -298,7 +300,6 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, ivp = nivp; } - idat += nb; count += nb; k += nb; i -= nb; diff --git a/freebsd/sys/powerpc/include/machine/spr.h b/freebsd/sys/powerpc/include/machine/spr.h index 24ae5fe9..f4769c86 100644 --- a/freebsd/sys/powerpc/include/machine/spr.h +++ b/freebsd/sys/powerpc/include/machine/spr.h @@ -97,6 +97,7 @@ #define SPR_RTCL_R 0x005 /* .6. 601 RTC Lower - Read */ #define SPR_LR 0x008 /* 468 Link Register */ #define SPR_CTR 0x009 /* 468 Count Register */ +#define SPR_DSCR 0x011 /* Data Stream Control Register */ #define SPR_DSISR 0x012 /* .68 DSI exception source */ #define DSISR_DIRECT 0x80000000 /* Direct-store error exception */ #define DSISR_NOTFOUND 0x40000000 /* Translation not found */ @@ -120,6 +121,20 @@ #define SPR_EIE 0x050 /* ..8 Exception Interrupt ??? */ #define SPR_EID 0x051 /* ..8 Exception Interrupt ??? */ #define SPR_NRI 0x052 /* ..8 Exception Interrupt ??? */ +#define SPR_FSCR 0x099 /* Facility Status and Control Register */ +#define FSCR_IC_MASK 0xFF00000000000000ULL /* FSCR[0:7] is Interrupt Cause */ +#define FSCR_IC_FP 0x0000000000000000ULL /* FP unavailable */ +#define FSCR_IC_VSX 0x0100000000000000ULL /* VSX unavailable */ +#define FSCR_IC_DSCR 0x0200000000000000ULL /* Access to the DSCR at SPRs 3 or 17 */ +#define FSCR_IC_PM 0x0300000000000000ULL /* Read or write access of a Performance Monitor SPR in group A */ +#define FSCR_IC_BHRB 0x0400000000000000ULL /* Execution of a BHRB Instruction */ +#define FSCR_IC_HTM 0x0500000000000000ULL /* Access to a Transactional Memory */ +/* Reserved 0x0600000000000000ULL */ +#define FSCR_IC_EBB 0x0700000000000000ULL /* Access to Event-Based Branch */ +#define FSCR_IC_TAR 0x0800000000000000ULL /* Access to Target Address Register */ +#define FSCR_IC_STOP 0x0900000000000000ULL /* Access to the 'stop' instruction in privileged non-hypervisor state */ +#define FSCR_IC_MSG 0x0A00000000000000ULL /* Access to 'msgsndp' or 'msgclrp' instructions */ +#define FSCR_IC_SCV 0x0C00000000000000ULL /* Execution of a 'scv' instruction */ #define SPR_USPRG0 0x100 /* 4.. User SPR General 0 */ #define SPR_VRSAVE 0x100 /* .6. AltiVec VRSAVE */ #define SPR_SPRG0 0x110 /* 468 SPR General 0 */ @@ -200,14 +215,6 @@ #define FSL_E300C3 0x8085 #define FSL_E300C4 0x8086 -#define SPR_LPCR 0x13e /* Logical Partitioning Control */ -#define LPCR_LPES 0x008 /* Bit 60 */ -#define LPCR_PECE_DRBL (1ULL << 16) /* Directed Privileged Doorbell */ -#define LPCR_PECE_HDRBL (1ULL << 15) /* Directed Hypervisor Doorbell */ -#define LPCR_PECE_EXT (1ULL << 14) /* External exceptions */ -#define LPCR_PECE_DECR (1ULL << 13) /* Decrementer exceptions */ -#define LPCR_PECE_ME (1ULL << 12) /* Machine Check and Hypervisor */ - /* Maintenance exceptions */ #define LPCR_PECE_WAKESET (LPCR_PECE_EXT | LPCR_PECE_DECR | LPCR_PECE_ME) #define SPR_EPCR 0x133 @@ -224,10 +231,20 @@ #define EPCR_PMGS 0x00200000 #define SPR_SPEFSCR 0x200 /* ..8 Signal Processing Engine FSCR. */ +#define SPR_HSRR0 0x13a +#define SPR_HSRR1 0x13b #define SPR_LPCR 0x13e /* Logical Partitioning Control */ -#define LPCR_LPES 0x008 /* Bit 60 */ +#define LPCR_LPES 0x008 /* Bit 60 */ +#define LPCR_HVICE 0x002 /* Hypervisor Virtualization Interrupt (Arch 3.0) */ +#define LPCR_PECE_DRBL (1ULL << 16) /* Directed Privileged Doorbell */ +#define LPCR_PECE_HDRBL (1ULL << 15) /* Directed Hypervisor Doorbell */ +#define LPCR_PECE_EXT (1ULL << 14) /* External exceptions */ +#define LPCR_PECE_DECR (1ULL << 13) /* Decrementer exceptions */ +#define LPCR_PECE_ME (1ULL << 12) /* Machine Check and Hypervisor */ + /* Maintenance exceptions */ #define SPR_LPID 0x13f /* Logical Partitioning Control */ +#define SPR_PTCR 0x1d0 /* Partition Table Control Register */ #define SPR_IBAT0U 0x210 /* .68 Instruction BAT Reg 0 Upper */ #define SPR_IBAT0U 0x210 /* .6. Instruction BAT Reg 0 Upper */ #define SPR_IBAT0L 0x211 /* .6. Instruction BAT Reg 0 Lower */ @@ -366,6 +383,7 @@ #define SPR_MD_CAM 0x338 /* ..8 IMMU CAM entry read */ #define SPR_MD_RAM0 0x339 /* ..8 IMMU RAM entry read reg 0 */ #define SPR_MD_RAM1 0x33a /* ..8 IMMU RAM entry read reg 1 */ +#define SPR_PSSCR 0x357 /* Processor Stop Status and Control Register (ISA 3.0) */ #define SPR_UMMCR2 0x3a0 /* .6. User Monitor Mode Control Register 2 */ #define SPR_UMMCR0 0x3a8 /* .6. User Monitor Mode Control Register 0 */ #define SPR_USIA 0x3ab /* .6. User Sampled Instruction Address */ diff --git a/freebsd/sys/sys/buf.h b/freebsd/sys/sys/buf.h index c9b2eeae..a099a972 100644 --- a/freebsd/sys/sys/buf.h +++ b/freebsd/sys/sys/buf.h @@ -479,6 +479,7 @@ buf_track(struct buf *bp, const char *location) #define GB_UNMAPPED 0x0008 /* Do not mmap buffer pages. */ #define GB_KVAALLOC 0x0010 /* But allocate KVA. */ #define GB_CKHASH 0x0020 /* If reading, calc checksum hash */ +#define GB_NOSPARSE 0x0040 /* Do not instantiate holes */ #ifdef _KERNEL extern int nbuf; /* The number of buffer headers */ @@ -540,6 +541,8 @@ struct buf * getpbuf(int *); struct buf *incore(struct bufobj *, daddr_t); struct buf *gbincore(struct bufobj *, daddr_t); struct buf *getblk(struct vnode *, daddr_t, int, int, int, int); +int getblkx(struct vnode *vp, daddr_t blkno, int size, int slpflag, + int slptimeo, int flags, struct buf **bpp); struct buf *geteblk(int, int); int bufwait(struct buf *); int bufwrite(struct buf *); diff --git a/freebsd/sys/sys/buf_ring.h b/freebsd/sys/sys/buf_ring.h index 0b633238..e8c69341 100644 --- a/freebsd/sys/sys/buf_ring.h +++ b/freebsd/sys/sys/buf_ring.h @@ -34,10 +34,6 @@ #include <machine/cpu.h> -#if defined(INVARIANTS) && !defined(DEBUG_BUFRING) -#define DEBUG_BUFRING 1 -#endif - #ifdef DEBUG_BUFRING #include <sys/lock.h> #include <sys/mutex.h> @@ -69,6 +65,12 @@ buf_ring_enqueue(struct buf_ring *br, void *buf) uint32_t prod_head, prod_next, cons_tail; #ifdef DEBUG_BUFRING int i; + + /* + * Note: It is possible to encounter an mbuf that was removed + * via drbr_peek(), and then re-added via drbr_putback() and + * trigger a spurious panic. + */ for (i = br->br_cons_head; i != br->br_prod_head; i = ((i + 1) & br->br_cons_mask)) if(br->br_ring[i] == buf) diff --git a/freebsd/sys/sys/bus.h b/freebsd/sys/sys/bus.h index d1d6bbb9..f3c54f36 100644 --- a/freebsd/sys/sys/bus.h +++ b/freebsd/sys/sys/bus.h @@ -46,7 +46,7 @@ */ struct u_businfo { int ub_version; /**< @brief interface version */ -#define BUS_USER_VERSION 1 +#define BUS_USER_VERSION 2 int ub_generation; /**< @brief generation count */ }; @@ -63,20 +63,23 @@ typedef enum device_state { /** * @brief Device information exported to userspace. + * The strings are placed one after the other, separated by NUL characters. + * Fields should be added after the last one and order maintained for compatibility */ +#define BUS_USER_BUFFER (3*1024) struct u_device { uintptr_t dv_handle; uintptr_t dv_parent; - - char dv_name[32]; /**< @brief Name of device in tree. */ - char dv_desc[32]; /**< @brief Driver description */ - char dv_drivername[32]; /**< @brief Driver name */ - char dv_pnpinfo[128]; /**< @brief Plug and play info */ - char dv_location[128]; /**< @brief Where is the device? */ uint32_t dv_devflags; /**< @brief API Flags for device */ uint16_t dv_flags; /**< @brief flags for dev state */ device_state_t dv_state; /**< @brief State of attachment */ - /* XXX more driver info? */ + char dv_fields[BUS_USER_BUFFER]; /**< @brief NUL terminated fields */ + /* name (name of the device in tree) */ + /* desc (driver description) */ + /* drivername (Name of driver without unit number) */ + /* pnpinfo (Plug and play information from bus) */ + /* location (Location of device on parent */ + /* NUL */ }; /* Flags exported via dv_flags. */ @@ -89,6 +92,7 @@ struct u_device { #define DF_EXTERNALSOFTC 0x40 /* softc not allocated by us */ #define DF_REBID 0x80 /* Can rebid after attach */ #define DF_SUSPENDED 0x100 /* Device is suspended. */ +#define DF_QUIET_CHILDREN 0x200 /* Default to quiet for all my children */ /** * @brief Device request structure used for ioctl's. @@ -584,6 +588,7 @@ device_state_t device_get_state(device_t dev); int device_get_unit(device_t dev); struct sysctl_ctx_list *device_get_sysctl_ctx(device_t dev); struct sysctl_oid *device_get_sysctl_tree(device_t dev); +int device_has_quiet_children(device_t dev); int device_is_alive(device_t dev); /* did probe succeed? */ int device_is_attached(device_t dev); /* did attach succeed? */ int device_is_enabled(device_t dev); @@ -597,6 +602,7 @@ int device_probe_and_attach(device_t dev); int device_probe_child(device_t bus, device_t dev); int device_quiesce(device_t dev); void device_quiet(device_t dev); +void device_quiet_children(device_t dev); void device_set_desc(device_t dev, const char* desc); void device_set_desc_copy(device_t dev, const char* desc); int device_set_devclass(device_t dev, const char *classname); diff --git a/freebsd/sys/sys/capsicum.h b/freebsd/sys/sys/capsicum.h index 847b4478..bf97d0b2 100644 --- a/freebsd/sys/sys/capsicum.h +++ b/freebsd/sys/sys/capsicum.h @@ -344,12 +344,115 @@ bool cap_rights_is_valid(const cap_rights_t *rights); cap_rights_t *cap_rights_merge(cap_rights_t *dst, const cap_rights_t *src); cap_rights_t *cap_rights_remove(cap_rights_t *dst, const cap_rights_t *src); bool cap_rights_contains(const cap_rights_t *big, const cap_rights_t *little); +void __cap_rights_sysinit(void *arg); __END_DECLS +struct cap_rights_init_args { + cap_rights_t *cria_rights; + uint64_t cria_value1; + uint64_t cria_value2; + uint64_t cria_value3; + uint64_t cria_value4; + uint64_t cria_value5; +}; + +#define CAP_RIGHTS_SYSINIT0(name, rights) \ + static struct cap_rights_init_args name##_args = { \ + &(rights) \ + }; \ + SYSINIT(name##_cap_rights_sysinit, SI_SUB_COPYRIGHT+1, SI_ORDER_ANY, \ + __cap_rights_sysinit, &name##_args); + +#define CAP_RIGHTS_SYSINIT1(name, rights, value1) \ + static struct cap_rights_init_args name##_args = { \ + &(rights), \ + (value1) \ + }; \ + SYSINIT(name##_cap_rights_sysinit, SI_SUB_COPYRIGHT+1, SI_ORDER_ANY, \ + __cap_rights_sysinit, &name##_args); + +#define CAP_RIGHTS_SYSINIT2(name, rights, value1, value2) \ + static struct cap_rights_init_args name##_args = { \ + &(rights), \ + (value1), \ + (value2) \ + }; \ + SYSINIT(name##_cap_rights_sysinit, SI_SUB_COPYRIGHT, SI_ORDER_ANY, \ + __cap_rights_sysinit, &name##_args); + +#define CAP_RIGHTS_SYSINIT3(name, rights, value1, value2, value3) \ + static struct cap_rights_init_args name##_args = { \ + &(rights), \ + (value1), \ + (value2), \ + (value3) \ + }; \ + SYSINIT(name##_cap_rights_sysinit, SI_SUB_COPYRIGHT, SI_ORDER_ANY, \ + __cap_rights_sysinit, &name##_args); + +#define CAP_RIGHTS_SYSINIT4(name, rights, value1, value2, value3, value4) \ + static struct cap_rights_init_args name##_args = { \ + &(rights), \ + (value1), \ + (value2), \ + (value3), \ + (value4) \ + }; \ + SYSINIT(name##_cap_rights_sysinit, SI_SUB_COPYRIGHT, SI_ORDER_ANY, \ + __cap_rights_sysinit, &name##_args); + +#define CAP_RIGHTS_DEFINE1(name, value) \ + __read_mostly cap_rights_t name; \ + CAP_RIGHTS_SYSINIT1(name, name, value); #ifdef _KERNEL #include <sys/systm.h> +extern cap_rights_t cap_accept_rights; +extern cap_rights_t cap_bind_rights; +extern cap_rights_t cap_connect_rights; +extern cap_rights_t cap_event_rights; +extern cap_rights_t cap_fchdir_rights; +extern cap_rights_t cap_fchflags_rights; +extern cap_rights_t cap_fchmod_rights; +extern cap_rights_t cap_fchown_rights; +extern cap_rights_t cap_fcntl_rights; +extern cap_rights_t cap_fexecve_rights; +extern cap_rights_t cap_flock_rights; +extern cap_rights_t cap_fpathconf_rights; +extern cap_rights_t cap_fstat_rights; +extern cap_rights_t cap_fstatfs_rights; +extern cap_rights_t cap_fsync_rights; +extern cap_rights_t cap_ftruncate_rights; +extern cap_rights_t cap_futimes_rights; +extern cap_rights_t cap_getpeername_rights; +extern cap_rights_t cap_getsockopt_rights; +extern cap_rights_t cap_getsockname_rights; +extern cap_rights_t cap_ioctl_rights; +extern cap_rights_t cap_linkat_source_rights; +extern cap_rights_t cap_linkat_target_rights; +extern cap_rights_t cap_listen_rights; +extern cap_rights_t cap_mkdirat_rights; +extern cap_rights_t cap_mkfifoat_rights; +extern cap_rights_t cap_mknodat_rights; +extern cap_rights_t cap_mmap_rights; +extern cap_rights_t cap_no_rights; +extern cap_rights_t cap_pdgetpid_rights; +extern cap_rights_t cap_pdkill_rights; +extern cap_rights_t cap_pread_rights; +extern cap_rights_t cap_pwrite_rights; +extern cap_rights_t cap_read_rights; +extern cap_rights_t cap_recv_rights; +extern cap_rights_t cap_renameat_source_rights; +extern cap_rights_t cap_renameat_target_rights; +extern cap_rights_t cap_seek_rights; +extern cap_rights_t cap_send_rights; +extern cap_rights_t cap_send_connect_rights; +extern cap_rights_t cap_setsockopt_rights; +extern cap_rights_t cap_shutdown_rights; +extern cap_rights_t cap_symlinkat_rights; +extern cap_rights_t cap_unlinkat_rights; +extern cap_rights_t cap_write_rights; #define IN_CAPABILITY_MODE(td) (((td)->td_ucred->cr_flags & CRED_FLAG_CAPMODE) != 0) @@ -363,20 +466,20 @@ int cap_check(const cap_rights_t *havep, const cap_rights_t *needp); /* * Convert capability rights into VM access flags. */ -u_char cap_rights_to_vmprot(cap_rights_t *havep); +u_char cap_rights_to_vmprot(const cap_rights_t *havep); /* * For the purposes of procstat(1) and similar tools, allow kern_descrip.c to * extract the rights from a capability. */ -cap_rights_t *cap_rights_fde(struct filedescent *fde); -cap_rights_t *cap_rights(struct filedesc *fdp, int fd); +const cap_rights_t *cap_rights_fde(const struct filedescent *fde); +const cap_rights_t *cap_rights(struct filedesc *fdp, int fd); int cap_ioctl_check(struct filedesc *fdp, int fd, u_long cmd); int cap_fcntl_check_fde(struct filedescent *fde, int cmd); int cap_fcntl_check(struct filedesc *fdp, int fd, int cmd); -extern int trap_enotcap; +extern bool trap_enotcap; #else /* !_KERNEL */ diff --git a/freebsd/sys/sys/ck.h b/freebsd/sys/sys/ck.h new file mode 100644 index 00000000..3bfce70c --- /dev/null +++ b/freebsd/sys/sys/ck.h @@ -0,0 +1,13 @@ +/* + * $FreeBSD$ + */ +#ifdef _KERNEL +#include <ck_queue.h> +#include <ck_epoch.h> +#else +#include <sys/queue.h> +#define CK_STAILQ_HEAD STAILQ_HEAD +#define CK_STAILQ_ENTRY STAILQ_ENTRY +#define CK_LIST_HEAD LIST_HEAD +#define CK_LIST_ENTRY LIST_ENTRY +#endif diff --git a/freebsd/sys/sys/conf.h b/freebsd/sys/sys/conf.h index 3980eba2..4ace162f 100644 --- a/freebsd/sys/sys/conf.h +++ b/freebsd/sys/sys/conf.h @@ -121,6 +121,8 @@ struct cdev { struct bio; struct buf; +struct dumperinfo; +struct kerneldumpheader; struct thread; struct uio; struct knote; @@ -151,6 +153,9 @@ typedef int dumper_t( vm_offset_t _physical, /* Physical address of virtual. */ off_t _offset, /* Byte-offset to write at. */ size_t _length); /* Number of bytes to dump. */ +typedef int dumper_start_t(struct dumperinfo *di); +typedef int dumper_hdr_t(struct dumperinfo *di, struct kerneldumpheader *kdh, + void *key, uint32_t keylen); #endif /* _KERNEL */ @@ -360,13 +365,18 @@ struct kerneldumpheader; struct dumperinfo { dumper_t *dumper; /* Dumping function. */ + dumper_start_t *dumper_start; /* Dumper callback for dump_start(). */ + dumper_hdr_t *dumper_hdr; /* Dumper callback for writing headers. */ void *priv; /* Private parts. */ u_int blocksize; /* Size of block in bytes. */ u_int maxiosize; /* Max size allowed for an individual I/O */ off_t mediaoffset; /* Initial offset in bytes. */ off_t mediasize; /* Space available in bytes. */ + + /* MI kernel dump state. */ void *blockbuf; /* Buffer for padding shorter dump blocks */ off_t dumpoff; /* Offset of ongoing kernel dump. */ + off_t origdumpoff; /* Starting dump offset. */ struct kerneldumpcrypto *kdcrypto; /* Kernel dump crypto. */ struct kerneldumpcomp *kdcomp; /* Kernel dump compression. */ }; @@ -381,6 +391,7 @@ int doadump(boolean_t); int set_dumper(struct dumperinfo *di, const char *devname, struct thread *td, uint8_t compression, uint8_t encryption, const uint8_t *key, uint32_t encryptedkeysize, const uint8_t *encryptedkey); +int clear_dumper(struct thread *td); int dump_start(struct dumperinfo *di, struct kerneldumpheader *kdh); int dump_append(struct dumperinfo *, void *, vm_offset_t, size_t); diff --git a/freebsd/sys/sys/cons.h b/freebsd/sys/sys/cons.h index aded0459..04784646 100644 --- a/freebsd/sys/sys/cons.h +++ b/freebsd/sys/sys/cons.h @@ -66,6 +66,8 @@ struct consdev_ops { /* grab console for exclusive kernel use */ cn_ungrab_t *cn_ungrab; /* ungrab console */ + cn_init_t *cn_resume; + /* set up console after sleep, optional */ }; struct consdev { @@ -105,8 +107,9 @@ extern struct tty *constty; /* Temporary virtual console. */ }; \ DATA_SET(cons_set, name) -#define CONSOLE_DRIVER(name) \ +#define CONSOLE_DRIVER(name, ...) \ static const struct consdev_ops name##_consdev_ops = { \ + /* Mandatory methods. */ \ .cn_probe = name##_cnprobe, \ .cn_init = name##_cninit, \ .cn_term = name##_cnterm, \ @@ -114,6 +117,8 @@ extern struct tty *constty; /* Temporary virtual console. */ .cn_putc = name##_cnputc, \ .cn_grab = name##_cngrab, \ .cn_ungrab = name##_cnungrab, \ + /* Optional fields. */ \ + __VA_ARGS__ \ }; \ CONSOLE_DEVICE(name##_consdev, name##_consdev_ops, NULL) @@ -126,6 +131,7 @@ void cnremove(struct consdev *); void cnselect(struct consdev *); void cngrab(void); void cnungrab(void); +void cnresume(void); int cncheckc(void); int cngetc(void); void cngets(char *, size_t, int); diff --git a/freebsd/sys/sys/filedesc.h b/freebsd/sys/sys/filedesc.h index 5a0b2db1..a3224f42 100644 --- a/freebsd/sys/sys/filedesc.h +++ b/freebsd/sys/sys/filedesc.h @@ -174,7 +174,7 @@ enum { struct thread; void filecaps_init(struct filecaps *fcaps); -int filecaps_copy(const struct filecaps *src, struct filecaps *dst, +bool filecaps_copy(const struct filecaps *src, struct filecaps *dst, bool locked); void filecaps_move(struct filecaps *src, struct filecaps *dst); #ifndef __rtems__ @@ -266,18 +266,19 @@ int fget_unlocked(struct filedesc *fdp, int fd, cap_rights_t *needrightsp, struct file **fpp, seq_t *seqp); #else /* __rtems__ */ static inline int -fget_unlocked(struct filedesc *fdp, int fd, cap_rights_t *needrightsp, - struct file **fpp, seq_t *seqp) +do_fget_unlocked(struct filedesc *fdp, int fd, struct file **fpp, seq_t *seqp) { struct file *fp; (void)fdp; - (void)needrightsp; (void)seqp; fp = rtems_bsd_get_file(fd); *fpp = fp; return (fp != NULL ? 0 : EBADF); } + +#define fget_unlocked(fdp, fd, needrightsp, fpp, seqp) \ + do_fget_unlocked(fdp, fd, fpp, seqp) #endif /* __rtems__ */ #ifndef __rtems__ diff --git a/freebsd/sys/sys/gtaskqueue.h b/freebsd/sys/sys/gtaskqueue.h index 41094603..c06ef503 100644 --- a/freebsd/sys/sys/gtaskqueue.h +++ b/freebsd/sys/sys/gtaskqueue.h @@ -54,13 +54,16 @@ void gtaskqueue_drain_all(struct gtaskqueue *queue); int grouptaskqueue_enqueue(struct gtaskqueue *queue, struct gtask *task); void taskqgroup_attach(struct taskqgroup *qgroup, struct grouptask *grptask, - void *uniq, int irq, char *name); + void *uniq, int irq, const char *name); int taskqgroup_attach_cpu(struct taskqgroup *qgroup, struct grouptask *grptask, - void *uniq, int cpu, int irq, char *name); + void *uniq, int cpu, int irq, const char *name); void taskqgroup_detach(struct taskqgroup *qgroup, struct grouptask *gtask); -struct taskqgroup *taskqgroup_create(char *name); +struct taskqgroup *taskqgroup_create(const char *name); void taskqgroup_destroy(struct taskqgroup *qgroup); int taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride); +void taskqgroup_config_gtask_init(void *ctx, struct grouptask *gtask, gtask_fn_t *fn, + const char *name); +void taskqgroup_config_gtask_deinit(struct grouptask *gtask); #define TASK_ENQUEUED 0x1 #define TASK_SKIP_WAKEUP 0x2 diff --git a/freebsd/sys/sys/jail.h b/freebsd/sys/sys/jail.h index 3bbbf5e0..c42964fe 100644 --- a/freebsd/sys/sys/jail.h +++ b/freebsd/sys/sys/jail.h @@ -216,7 +216,10 @@ struct prison_racct { #define PR_IP6 0x04000000 /* IPv6 restricted or disabled */ /* by this jail or an ancestor */ -/* Flags for pr_allow */ +/* + * Flags for pr_allow + * Bits not noted here may be used for dynamic allow.mount.xxxfs. + */ #define PR_ALLOW_SET_HOSTNAME 0x00000001 #define PR_ALLOW_SYSVIPC 0x00000002 #define PR_ALLOW_RAW_SOCKETS 0x00000004 @@ -224,17 +227,9 @@ struct prison_racct { #define PR_ALLOW_MOUNT 0x00000010 #define PR_ALLOW_QUOTAS 0x00000020 #define PR_ALLOW_SOCKET_AF 0x00000040 -#define PR_ALLOW_MOUNT_DEVFS 0x00000080 -#define PR_ALLOW_MOUNT_NULLFS 0x00000100 -#define PR_ALLOW_MOUNT_ZFS 0x00000200 -#define PR_ALLOW_MOUNT_PROCFS 0x00000400 -#define PR_ALLOW_MOUNT_TMPFS 0x00000800 -#define PR_ALLOW_MOUNT_FDESCFS 0x00001000 -#define PR_ALLOW_MOUNT_LINPROCFS 0x00002000 -#define PR_ALLOW_MOUNT_LINSYSFS 0x00004000 #define PR_ALLOW_RESERVED_PORTS 0x00008000 #define PR_ALLOW_KMEM_ACCESS 0x00010000 /* reserved, not used yet */ -#define PR_ALLOW_ALL 0x0001ffff +#define PR_ALLOW_ALL_STATIC 0x0001807f /* * OSD methods @@ -364,6 +359,7 @@ struct ucred; struct mount; struct sockaddr; struct statfs; +struct vfsconf; int jailed(struct ucred *cred); int jailed_without_vnet(struct ucred *); void getcredhostname(struct ucred *, char *, size_t); @@ -421,6 +417,7 @@ int prison_if(struct ucred *cred, struct sockaddr *sa); char *prison_name(struct prison *, struct prison *); int prison_priv_check(struct ucred *cred, int priv); int sysctl_jail_param(SYSCTL_HANDLER_ARGS); +void prison_add_vfs(struct vfsconf *vfsp); void prison_racct_foreach(void (*callback)(struct racct *racct, void *arg2, void *arg3), void (*pre)(void), void (*post)(void), void *arg2, void *arg3); diff --git a/freebsd/sys/sys/linker.h b/freebsd/sys/sys/linker.h index 6d560574..21c5a41e 100644 --- a/freebsd/sys/sys/linker.h +++ b/freebsd/sys/sys/linker.h @@ -273,11 +273,16 @@ extern int kld_debug; typedef int elf_lookup_fn(linker_file_t, Elf_Size, int, Elf_Addr *); /* Support functions */ -int elf_reloc(linker_file_t _lf, Elf_Addr base, const void *_rel, int _type, elf_lookup_fn _lu); -int elf_reloc_local(linker_file_t _lf, Elf_Addr base, const void *_rel, int _type, elf_lookup_fn _lu); +int elf_reloc(linker_file_t _lf, Elf_Addr base, const void *_rel, + int _type, elf_lookup_fn _lu); +int elf_reloc_ifunc(linker_file_t _lf, Elf_Addr base, const void *_rel, + int _type, elf_lookup_fn _lu); +int elf_reloc_local(linker_file_t _lf, Elf_Addr base, const void *_rel, + int _type, elf_lookup_fn _lu); Elf_Addr elf_relocaddr(linker_file_t _lf, Elf_Addr addr); const Elf_Sym *elf_get_sym(linker_file_t _lf, Elf_Size _symidx); const char *elf_get_symname(linker_file_t _lf, Elf_Size _symidx); +void link_elf_ireloc(caddr_t kmdp); typedef struct linker_ctf { const uint8_t *ctftab; /* Decompressed CTF data. */ diff --git a/freebsd/sys/sys/lockstat.h b/freebsd/sys/sys/lockstat.h index 6167a7d1..9a6674fa 100644 --- a/freebsd/sys/sys/lockstat.h +++ b/freebsd/sys/sys/lockstat.h @@ -70,7 +70,7 @@ SDT_PROBE_DECLARE(lockstat, , , thread__spin); #define LOCKSTAT_WRITER 0 #define LOCKSTAT_READER 1 -extern volatile int lockstat_enabled; +extern volatile bool lockstat_enabled; #ifdef KDTRACE_HOOKS diff --git a/freebsd/sys/sys/malloc.h b/freebsd/sys/sys/malloc.h index 1920ff69..8c66fe81 100644 --- a/freebsd/sys/sys/malloc.h +++ b/freebsd/sys/sys/malloc.h @@ -59,6 +59,13 @@ #define M_MAGIC 877983977 /* time when first defined :-) */ +#ifdef INVARIANTS +#define M_ZERO_INVARIANTS M_ZERO +#else +#define M_ZERO_INVARIANTS 0 +#endif + + /* * Two malloc type structures are present: malloc_type, which is used by a * type owner to declare the type, and malloc_type_internal, which holds diff --git a/freebsd/sys/sys/mbuf.h b/freebsd/sys/sys/mbuf.h index 4a91b794..4d2a3223 100644 --- a/freebsd/sys/sys/mbuf.h +++ b/freebsd/sys/sys/mbuf.h @@ -196,6 +196,11 @@ struct pkthdr { #define lro_nsegs tso_segsz #define csum_phsum PH_per.sixteen[2] #define csum_data PH_per.thirtytwo[1] +#define pace_thoff PH_loc.sixteen[0] +#define pace_tlen PH_loc.sixteen[1] +#define pace_drphdrlen PH_loc.sixteen[2] +#define pace_tos PH_loc.eight[6] +#define pace_lock PH_loc.eight[7] /* * Description of external storage mapped into mbuf; valid only if M_EXT is @@ -1375,5 +1380,12 @@ mbuf_tstmp2timespec(struct mbuf *m, struct timespec *ts) } #endif +#ifdef NETDUMP +/* Invoked from the netdump client code. */ +void netdump_mbuf_drain(void); +void netdump_mbuf_dump(void); +void netdump_mbuf_reinit(int nmbuf, int nclust, int clsize); +#endif + #endif /* _KERNEL */ #endif /* !_SYS_MBUF_H_ */ diff --git a/freebsd/sys/sys/module.h b/freebsd/sys/sys/module.h index c3f8dc3c..6799b179 100644 --- a/freebsd/sys/sys/module.h +++ b/freebsd/sys/sys/module.h @@ -141,7 +141,7 @@ struct mod_pnp_match_info #define DECLARE_MODULE_WITH_MAXVER(name, data, sub, order, maxver) \ MODULE_DEPEND(name, kernel, __FreeBSD_version, \ - __FreeBSD_version, maxver); \ + __FreeBSD_version, maxver); \ MODULE_METADATA(_md_##name, MDT_MODULE, &data, __XSTRING(name));\ SYSINIT(name##module, sub, order, module_register_init, &data); \ struct __hack @@ -156,7 +156,7 @@ struct mod_pnp_match_info * Use it for modules that use kernel interfaces that are not stable * even on STABLE/X branches. */ -#define DECLARE_MODULE_TIED(name, data, sub, order) \ +#define DECLARE_MODULE_TIED(name, data, sub, order) \ DECLARE_MODULE_WITH_MAXVER(name, data, sub, order, __FreeBSD_version) #define MODULE_VERSION_CONCAT(module, version) _##module##_version diff --git a/freebsd/sys/sys/mount.h b/freebsd/sys/sys/mount.h index 9d499004..c4f4bebf 100644 --- a/freebsd/sys/sys/mount.h +++ b/freebsd/sys/sys/mount.h @@ -516,6 +516,7 @@ struct vfsconf { int vfc_typenum; /* historic filesystem type number */ int vfc_refcount; /* number mounted of this type */ int vfc_flags; /* permanent flags */ + int vfc_prison_flag; /* prison allow.mount.* flag */ struct vfsoptdecl *vfc_opts; /* mount options */ TAILQ_ENTRY(vfsconf) vfc_list; /* list of vfscons */ }; @@ -851,7 +852,8 @@ vfs_statfs_t __vfs_statfs; */ #define VFS_VERSION_00 0x19660120 #define VFS_VERSION_01 0x20121030 -#define VFS_VERSION VFS_VERSION_01 +#define VFS_VERSION_02 0x20180504 +#define VFS_VERSION VFS_VERSION_02 #define VFS_SET(vfsops, fsname, flags) \ static struct vfsconf fsname ## _vfsconf = { \ diff --git a/freebsd/sys/sys/proc.h b/freebsd/sys/sys/proc.h index cef3ae2e..36ed69cc 100644 --- a/freebsd/sys/sys/proc.h +++ b/freebsd/sys/sys/proc.h @@ -76,6 +76,18 @@ /* + * A section object may be passed to every begin-end pair to allow for + * forward progress guarantees with-in prolonged active sections. + * + * We can't include ck_epoch.h so we define our own variant here and + * then CTASSERT that it's the same size in subr_epoch.c + */ +struct epoch_section { + unsigned int bucket; +}; +typedef struct epoch_section epoch_section_t; + +/* * One structure allocated per session. * * List of locks @@ -263,7 +275,8 @@ struct thread { u_char td_lend_user_pri; /* (t) Lend user pri. */ /* Cleared during fork1() */ -#define td_startzero td_flags +#define td_startzero td_epochnest + u_char td_epochnest; /* (k) Epoch nest counter. */ int td_flags; /* (t) TDF_* flags. */ int td_inhibitors; /* (t) Why can not run. */ int td_pflags; /* (k) Private thread (TDP_*) flags. */ @@ -280,6 +293,7 @@ struct thread { u_char td_tsqueue; /* (t) Turnstile queue blocked on. */ short td_locks; /* (k) Debug: count of non-spin locks */ short td_rw_rlocks; /* (k) Count of rwlock read locks. */ + short td_sx_slocks; /* (k) Count of sx shared locks. */ short td_lk_slocks; /* (k) Count of lockmgr shared locks. */ short td_stopsched; /* (k) Scheduler stopped. */ struct turnstile *td_blocked; /* (t) Lock thread is blocked on. */ @@ -340,6 +354,7 @@ struct thread { u_char td_pri_class; /* (t) Scheduling class. */ u_char td_user_pri; /* (t) User pri from estcpu and nice. */ u_char td_base_user_pri; /* (t) Base user pri */ + u_char td_pre_epoch_prio; /* (k) User pri on entry to epoch */ uintptr_t td_rb_list; /* (k) Robust list head. */ uintptr_t td_rbp_list; /* (k) Robust priv list head. */ uintptr_t td_rb_inact; /* (k) Current in-action mutex loc. */ @@ -369,6 +384,7 @@ struct thread { #ifndef __rtems__ #define td_retval td_uretoff.tdu_retval u_int td_cowgen; /* (k) Generation of COW pointers. */ + /* LP64 hole */ struct callout td_slpcallout; /* (h) Callout for sleep. */ struct trapframe *td_frame; /* (k) */ struct vm_object *td_kstack_obj;/* (a) Kstack object. */ @@ -380,16 +396,20 @@ struct thread { struct lpohead td_lprof[2]; /* (a) lock profiling objects. */ struct kdtrace_thread *td_dtrace; /* (*) DTrace-specific data. */ int td_errno; /* Error returned by last syscall. */ + /* LP64 hole */ struct vnet *td_vnet; /* (k) Effective vnet. */ const char *td_vnet_lpush; /* (k) Debugging vnet push / pop. */ struct trapframe *td_intr_frame;/* (k) Frame of the current irq */ struct proc *td_rfppwait_p; /* (k) The vforked child */ struct vm_page **td_ma; /* (k) uio pages held */ int td_ma_cnt; /* (k) size of *td_ma */ + /* LP64 hole */ void *td_emuldata; /* Emulator state data */ int td_lastcpu; /* (t) Last cpu we were on. */ int td_oncpu; /* (t) Which cpu we are on. */ void *td_lkpi_task; /* LinuxKPI task struct pointer */ + TAILQ_ENTRY(thread) td_epochq; /* (t) Epoch queue. */ + epoch_section_t td_epoch_section; /* (t) epoch section object */ #endif /* __rtems__ */ }; @@ -485,6 +505,7 @@ do { \ #define TDB_EXIT 0x00000400 /* Exiting LWP indicator for ptrace() */ #define TDB_VFORK 0x00000800 /* vfork indicator for ptrace() */ #define TDB_FSTP 0x00001000 /* The thread is PT_ATTACH leader */ +#define TDB_STEP 0x00002000 /* (x86) PSL_T set for PT_STEP */ /* * "Private" flags kept in td_pflags: @@ -669,6 +690,7 @@ struct proc { u_int p_treeflag; /* (e) P_TREE flags */ int p_pendingexits; /* (c) Count of pending thread exits. */ struct filemon *p_filemon; /* (c) filemon-specific data. */ + int p_pdeathsig; /* (c) Signal from parent on exit. */ /* End area that is zeroed on creation. */ #define p_endzero p_magic @@ -713,7 +735,7 @@ struct proc { struct racct *p_racct; /* (b) Resource accounting. */ int p_throttled; /* (c) Flag for racct pcpu throttling */ /* - * An orphan is the child that has beed re-parented to the + * An orphan is the child that has been re-parented to the * debugger as a result of attaching to it. Need to keep * track of them for parent to be able to collect the exit * status of what used to be children. diff --git a/freebsd/sys/sys/random.h b/freebsd/sys/sys/random.h index 69d377f5..c717a686 100644 --- a/freebsd/sys/sys/random.h +++ b/freebsd/sys/sys/random.h @@ -116,9 +116,38 @@ enum random_entropy_source { #define RANDOM_CACHED_SKIP_START 256 #if defined(DEV_RANDOM) -void random_harvest_queue(const void *, u_int, u_int, enum random_entropy_source); -void random_harvest_fast(const void *, u_int, u_int, enum random_entropy_source); -void random_harvest_direct(const void *, u_int, u_int, enum random_entropy_source); +extern u_int hc_source_mask; +void random_harvest_queue_(const void *, u_int, u_int, enum random_entropy_source); +void random_harvest_fast_(const void *, u_int, u_int); +void random_harvest_direct_(const void *, u_int, u_int, enum random_entropy_source); + +static __inline void +random_harvest_queue(const void *entropy, u_int size, u_int bits, + enum random_entropy_source origin) +{ + + if (hc_source_mask & (1 << origin)) + random_harvest_queue_(entropy, size, bits, origin); +} + +static __inline void +random_harvest_fast(const void *entropy, u_int size, u_int bits, + enum random_entropy_source origin) +{ + + if (hc_source_mask & (1 << origin)) + random_harvest_fast_(entropy, size, bits); +} + +static __inline void +random_harvest_direct(const void *entropy, u_int size, u_int bits, + enum random_entropy_source origin) +{ + + if (hc_source_mask & (1 << origin)) + random_harvest_direct_(entropy, size, bits, origin); +} + void random_harvest_register_source(enum random_entropy_source); void random_harvest_deregister_source(enum random_entropy_source); #else @@ -135,6 +164,13 @@ void random_harvest_deregister_source(enum random_entropy_source); #define random_harvest_fast_uma(a, b, c, d) do {} while (0) #endif /* defined(RANDOM_ENABLE_UMA) */ +#if defined(RANDOM_ENABLE_ETHER) +#define random_harvest_queue_ether(a, b, c) random_harvest_queue(a, b, c, RANDOM_NET_ETHER) +#else /* !defined(RANDOM_ENABLE_ETHER) */ +#define random_harvest_queue_ether(a, b, c) do {} while (0) +#endif /* defined(RANDOM_ENABLE_ETHER) */ + + #endif /* _KERNEL */ #define GRND_NONBLOCK 0x1 diff --git a/freebsd/sys/sys/resourcevar.h b/freebsd/sys/sys/resourcevar.h index b84bc432..9301c3e0 100644 --- a/freebsd/sys/sys/resourcevar.h +++ b/freebsd/sys/sys/resourcevar.h @@ -143,7 +143,6 @@ rtems_bsd_chgsbsize(u_int *hiwat, u_int to) #endif /* __rtems__ */ int chgptscnt(struct uidinfo *uip, int diff, rlim_t maxval); int chgumtxcnt(struct uidinfo *uip, int diff, rlim_t maxval); -int fuswintr(void *base); int kern_proc_setrlimit(struct thread *td, struct proc *p, u_int which, struct rlimit *limp); struct plimit @@ -167,7 +166,6 @@ void rufetchcalc(struct proc *p, struct rusage *ru, struct timeval *up, struct timeval *sp); void rufetchtd(struct thread *td, struct rusage *ru); void ruxagg(struct proc *p, struct thread *td); -int suswintr(void *base, int word); struct uidinfo *uifind(uid_t uid); void uifree(struct uidinfo *uip); diff --git a/freebsd/sys/sys/stdint.h b/freebsd/sys/sys/stdint.h index 98915f72..44384d24 100644 --- a/freebsd/sys/sys/stdint.h +++ b/freebsd/sys/sys/stdint.h @@ -36,8 +36,10 @@ #ifndef __rtems__ #include <machine/_stdint.h> -#endif /* __rtems__ */ #include <sys/_stdint.h> +#else /* __rtems__ */ +#include <stdint.h> +#endif /* __rtems__ */ #ifndef __rtems__ typedef __int_least8_t int_least8_t; diff --git a/freebsd/sys/sys/sx.h b/freebsd/sys/sys/sx.h index 33143057..566137bd 100644 --- a/freebsd/sys/sys/sx.h +++ b/freebsd/sys/sys/sx.h @@ -77,12 +77,14 @@ #define SX_LOCK_SHARED_WAITERS 0x02 #define SX_LOCK_EXCLUSIVE_WAITERS 0x04 #define SX_LOCK_RECURSED 0x08 +#define SX_LOCK_WRITE_SPINNER 0x10 #define SX_LOCK_FLAGMASK \ (SX_LOCK_SHARED | SX_LOCK_SHARED_WAITERS | \ - SX_LOCK_EXCLUSIVE_WAITERS | SX_LOCK_RECURSED) + SX_LOCK_EXCLUSIVE_WAITERS | SX_LOCK_RECURSED | SX_LOCK_WRITE_SPINNER) +#define SX_LOCK_WAITERS (SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS) #define SX_OWNER(x) ((x) & ~SX_LOCK_FLAGMASK) -#define SX_SHARERS_SHIFT 4 +#define SX_SHARERS_SHIFT 5 #define SX_SHARERS(x) (SX_OWNER(x) >> SX_SHARERS_SHIFT) #define SX_SHARERS_LOCK(x) \ ((x) << SX_SHARERS_SHIFT | SX_LOCK_SHARED) diff --git a/freebsd/sys/sys/sysctl.h b/freebsd/sys/sys/sysctl.h index 874d226f..995baf64 100644 --- a/freebsd/sys/sys/sysctl.h +++ b/freebsd/sys/sys/sysctl.h @@ -38,7 +38,9 @@ #ifndef _SYS_SYSCTL_H_ #define _SYS_SYSCTL_H_ +#ifdef _KERNEL #include <sys/queue.h> +#endif struct thread; /* @@ -1053,9 +1055,6 @@ SYSCTL_DECL(_compat); SYSCTL_DECL(_regression); SYSCTL_DECL(_security); SYSCTL_DECL(_security_bsd); -#ifdef EXT_RESOURCES -SYSCTL_DECL(_clock); -#endif extern char machine[]; extern char osrelease[]; diff --git a/freebsd/sys/sys/sysproto.h b/freebsd/sys/sys/sysproto.h index f70d820c..b328cb51 100644 --- a/freebsd/sys/sys/sysproto.h +++ b/freebsd/sys/sys/sysproto.h @@ -271,9 +271,6 @@ struct sbrk_args { struct sstk_args { char incr_l_[PADL_(int)]; int incr; char incr_r_[PADR_(int)]; }; -struct ovadvise_args { - char anom_l_[PADL_(int)]; int anom; char anom_r_[PADR_(int)]; -}; struct munmap_args { char addr_l_[PADL_(void *)]; void * addr; char addr_r_[PADR_(void *)]; char len_l_[PADL_(size_t)]; size_t len; char len_r_[PADR_(size_t)]; @@ -1875,7 +1872,6 @@ int sys_msync(struct thread *, struct msync_args *); int sys_vfork(struct thread *, struct vfork_args *); int sys_sbrk(struct thread *, struct sbrk_args *); int sys_sstk(struct thread *, struct sstk_args *); -int sys_ovadvise(struct thread *, struct ovadvise_args *); int sys_munmap(struct thread *, struct munmap_args *); int sys_mprotect(struct thread *, struct mprotect_args *); int sys_madvise(struct thread *, struct madvise_args *); @@ -2540,6 +2536,9 @@ struct freebsd11_mknod_args { char mode_l_[PADL_(int)]; int mode; char mode_r_[PADR_(int)]; char dev_l_[PADL_(int)]; int dev; char dev_r_[PADR_(int)]; }; +struct freebsd11_vadvise_args { + char anom_l_[PADL_(int)]; int anom; char anom_r_[PADR_(int)]; +}; struct freebsd11_stat_args { char path_l_[PADL_(char *)]; char * path; char path_r_[PADR_(char *)]; char ub_l_[PADL_(struct freebsd11_stat *)]; struct freebsd11_stat * ub; char ub_r_[PADR_(struct freebsd11_stat *)]; @@ -2617,6 +2616,7 @@ struct freebsd11_mknodat_args { char dev_l_[PADL_(uint32_t)]; uint32_t dev; char dev_r_[PADR_(uint32_t)]; }; int freebsd11_mknod(struct thread *, struct freebsd11_mknod_args *); +int freebsd11_vadvise(struct thread *, struct freebsd11_vadvise_args *); int freebsd11_stat(struct thread *, struct freebsd11_stat_args *); int freebsd11_fstat(struct thread *, struct freebsd11_fstat_args *); int freebsd11_lstat(struct thread *, struct freebsd11_lstat_args *); @@ -2705,7 +2705,7 @@ int freebsd11_mknodat(struct thread *, struct freebsd11_mknodat_args *); #define SYS_AUE_sbrk AUE_SBRK #define SYS_AUE_sstk AUE_SSTK #define SYS_AUE_ommap AUE_MMAP -#define SYS_AUE_vadvise AUE_O_VADVISE +#define SYS_AUE_freebsd11_vadvise AUE_O_VADVISE #define SYS_AUE_munmap AUE_MUNMAP #define SYS_AUE_mprotect AUE_MPROTECT #define SYS_AUE_madvise AUE_MADVISE diff --git a/freebsd/sys/sys/systm.h b/freebsd/sys/sys/systm.h index 98efffd0..9c21f589 100644 --- a/freebsd/sys/sys/systm.h +++ b/freebsd/sys/sys/systm.h @@ -331,6 +331,12 @@ void hexdump(const void *ptr, int length, const char *hdr, int flags); #define ovbcopy(f, t, l) bcopy((f), (t), (l)) #ifndef __rtems__ void bcopy(const void * _Nonnull from, void * _Nonnull to, size_t len); +#define bcopy(from, to, len) ({ \ + if (__builtin_constant_p(len) && (len) <= 64) \ + __builtin_memmove((to), (from), (len)); \ + else \ + bcopy((from), (to), (len)); \ +}) void bzero(void * _Nonnull buf, size_t len); #define bzero(buf, len) ({ \ if (__builtin_constant_p(len) && (len) <= 64) \ @@ -345,6 +351,7 @@ void bzero(void * _Nonnull buf, size_t len); void explicit_bzero(void * _Nonnull, size_t); void *memcpy(void * _Nonnull to, const void * _Nonnull from, size_t len); +#define memcpy(to, from, len) __builtin_memcpy(to, from, len) void *memmove(void * _Nonnull dest, const void * _Nonnull src, size_t n); #ifndef __rtems__ @@ -462,6 +469,8 @@ void startprofclock(struct proc *); void stopprofclock(struct proc *); void cpu_startprofclock(void); void cpu_stopprofclock(void); +void suspendclock(void); +void resumeclock(void); sbintime_t cpu_idleclock(void); void cpu_activeclock(void); void cpu_new_callout(int cpu, sbintime_t bt, sbintime_t bt_opt); diff --git a/freebsd/sys/sys/unpcb.h b/freebsd/sys/sys/unpcb.h index 1ab3457d..d80b1384 100644 --- a/freebsd/sys/sys/unpcb.h +++ b/freebsd/sys/sys/unpcb.h @@ -69,27 +69,29 @@ typedef uint64_t unp_gen_t; LIST_HEAD(unp_head, unpcb); struct unpcb { - LIST_ENTRY(unpcb) unp_link; /* glue on list of all PCBs */ + /* Cache line 1 */ + struct mtx unp_mtx; /* mutex */ + struct unpcb *unp_conn; /* control block of connected socket */ + volatile u_int unp_refcount; + short unp_flags; /* flags */ + short unp_gcflag; /* Garbage collector flags. */ + struct sockaddr_un *unp_addr; /* bound address of socket */ struct socket *unp_socket; /* pointer back to socket */ - struct file *unp_file; /* back-pointer to file for gc. */ + /* Cache line 2 */ #ifndef __rtems__ struct vnode *unp_vnode; /* if associated with file */ #else /* __rtems__ */ void *unp_vnode; /* if associated with file */ #endif /* __rtems__ */ - ino_t unp_ino; /* fake inode number */ - struct unpcb *unp_conn; /* control block of connected socket */ - struct unp_head unp_refs; /* referencing socket linked list */ + struct xucred unp_peercred; /* peer credentials, if applicable */ LIST_ENTRY(unpcb) unp_reflink; /* link in unp_refs list */ - struct sockaddr_un *unp_addr; /* bound address of socket */ + LIST_ENTRY(unpcb) unp_link; /* glue on list of all PCBs */ + struct unp_head unp_refs; /* referencing socket linked list */ unp_gen_t unp_gencnt; /* generation count of this instance */ - short unp_flags; /* flags */ - short unp_gcflag; /* Garbage collector flags. */ - struct xucred unp_peercred; /* peer credentials, if applicable */ - u_int unp_refcount; + struct file *unp_file; /* back-pointer to file for gc. */ u_int unp_msgcount; /* references from message queue */ - struct mtx unp_mtx; /* mutex */ -}; + ino_t unp_ino; /* fake inode number */ +} __aligned(CACHE_LINE_SIZE); /* * Flags in unp_flags. diff --git a/freebsd/sys/sys/vmmeter.h b/freebsd/sys/sys/vmmeter.h index 408cd862..3c570b0a 100644 --- a/freebsd/sys/sys/vmmeter.h +++ b/freebsd/sys/sys/vmmeter.h @@ -73,11 +73,8 @@ struct vmtotal { /* * System wide statistics counters. * Locking: - * a - locked by atomic operations * c - constant after initialization - * f - locked by vm_page_queue_free_mtx * p - uses counter(9) - * q - changes are synchronized by the corresponding vm_pagequeue lock */ struct vmmeter { /* diff --git a/freebsd/sys/vm/uma.h b/freebsd/sys/vm/uma.h index 935c5aa7..d71f0ee3 100644 --- a/freebsd/sys/vm/uma.h +++ b/freebsd/sys/vm/uma.h @@ -265,8 +265,8 @@ uma_zone_t uma_zcache_create(char *name, int size, uma_ctor ctor, uma_dtor dtor, * information in the vm_page. */ #define UMA_ZONE_SECONDARY 0x0200 /* Zone is a Secondary Zone */ -/* 0x0400 Unused */ -#define UMA_ZONE_MAXBUCKET 0x0800 /* Use largest buckets */ +#define UMA_ZONE_NOBUCKET 0x0400 /* Do not use buckets. */ +#define UMA_ZONE_MAXBUCKET 0x0800 /* Use largest buckets. */ #define UMA_ZONE_CACHESPREAD 0x1000 /* * Spread memory start locations across * all possible cache lines. May @@ -286,6 +286,10 @@ uma_zone_t uma_zcache_create(char *name, int size, uma_ctor ctor, uma_dtor dtor, * NUMA aware Zone. Implements a best * effort first-touch policy. */ +#define UMA_ZONE_NOBUCKETCACHE 0x20000 /* + * Don't cache full buckets. Limit + * UMA to per-cpu state. + */ /* * These flags are shared between the keg and zone. In zones wishing to add diff --git a/freebsd/sys/vm/uma_core.c b/freebsd/sys/vm/uma_core.c index fdf7dc35..b8145c72 100644 --- a/freebsd/sys/vm/uma_core.c +++ b/freebsd/sys/vm/uma_core.c @@ -1364,7 +1364,15 @@ keg_small_init(uma_keg_t keg) else shsize = sizeof(struct uma_slab); - keg->uk_ipers = (slabsize - shsize) / rsize; + if (rsize <= slabsize - shsize) + keg->uk_ipers = (slabsize - shsize) / rsize; + else { + /* Handle special case when we have 1 item per slab, so + * alignment requirement can be relaxed. */ + KASSERT(keg->uk_size <= slabsize - shsize, + ("%s: size %u greater than slab", __func__, keg->uk_size)); + keg->uk_ipers = 1; + } KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE, ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers)); @@ -1547,7 +1555,7 @@ keg_ctor(void *mem, int size, void *udata, int flags) if (keg->uk_flags & UMA_ZONE_CACHESPREAD) { keg_cachespread_init(keg); } else { - if (keg->uk_size > (UMA_SLAB_SIZE - sizeof(struct uma_slab))) + if (keg->uk_size > UMA_SLAB_SPACE) keg_large_init(keg); else keg_small_init(keg); @@ -1751,10 +1759,15 @@ zone_ctor(void *mem, int size, void *udata, int flags) } out: - if ((arg->flags & UMA_ZONE_MAXBUCKET) == 0) - zone->uz_count = bucket_select(zone->uz_size); - else + KASSERT((arg->flags & (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET)) != + (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET), + ("Invalid zone flag combination")); + if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0) zone->uz_count = BUCKET_MAX; + else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0) + zone->uz_count = 0; + else + zone->uz_count = bucket_select(zone->uz_size); zone->uz_count_min = zone->uz_count; return (0); @@ -1901,9 +1914,11 @@ uma_startup_count(int vm_zones) #endif /* Memory for the rest of startup zones, UMA and VM, ... */ - if (zsize > UMA_SLAB_SIZE) + if (zsize > UMA_SLAB_SPACE) pages += (zones + vm_zones) * howmany(roundup2(zsize, UMA_BOOT_ALIGN), UMA_SLAB_SIZE); + else if (roundup2(zsize, UMA_BOOT_ALIGN) > UMA_SLAB_SPACE) + pages += zones; else pages += howmany(zones, UMA_SLAB_SPACE / roundup2(zsize, UMA_BOOT_ALIGN)); @@ -2444,14 +2459,6 @@ zalloc_start: cpu = curcpu; cache = &zone->uz_cpu[cpu]; - /* - * Since we have locked the zone we may as well send back our stats. - */ - atomic_add_long(&zone->uz_allocs, cache->uc_allocs); - atomic_add_long(&zone->uz_frees, cache->uc_frees); - cache->uc_allocs = 0; - cache->uc_frees = 0; - /* See if we lost the race to fill the cache. */ if (cache->uc_allocbucket != NULL) { ZONE_UNLOCK(zone); @@ -3103,7 +3110,13 @@ zfree_start: /* ub_cnt is pointing to the last free item */ KASSERT(bucket->ub_cnt != 0, ("uma_zfree: Attempting to insert an empty bucket onto the full list.\n")); - LIST_INSERT_HEAD(&zdom->uzd_buckets, bucket, ub_link); + if ((zone->uz_flags & UMA_ZONE_NOBUCKETCACHE) != 0) { + ZONE_UNLOCK(zone); + bucket_drain(zone, bucket); + bucket_free(zone, bucket, udata); + goto zfree_restart; + } else + LIST_INSERT_HEAD(&zdom->uzd_buckets, bucket, ub_link); } /* diff --git a/freebsd/sys/vm/uma_int.h b/freebsd/sys/vm/uma_int.h index 53d65dac..8d58fa33 100644 --- a/freebsd/sys/vm/uma_int.h +++ b/freebsd/sys/vm/uma_int.h @@ -177,7 +177,7 @@ struct uma_hash { * align field or structure to cache line */ #if defined(__amd64__) -#define UMA_ALIGN __aligned(CACHE_LINE_SIZE) +#define UMA_ALIGN __aligned(128) #else #define UMA_ALIGN #endif diff --git a/freebsd/usr.bin/netstat/ipsec.c b/freebsd/usr.bin/netstat/ipsec.c index ba0fb483..97111b4b 100644 --- a/freebsd/usr.bin/netstat/ipsec.c +++ b/freebsd/usr.bin/netstat/ipsec.c @@ -203,6 +203,8 @@ print_ipsecstats(const struct ipsecstat *ipsecstat) #define p(f, m) if (ipsecstat->f || sflag <= 1) \ xo_emit(m, (uintmax_t)ipsecstat->f, plural(ipsecstat->f)) +#define p2(f, m) if (ipsecstat->f || sflag <= 1) \ + xo_emit(m, (uintmax_t)ipsecstat->f, plurales(ipsecstat->f)) p(ips_in_polvio, "\t{:dropped-policy-violation/%ju} " "{N:/inbound packet%s violated process security policy}\n"); @@ -222,14 +224,15 @@ print_ipsecstats(const struct ipsecstat *ipsecstat) "{N:/invalid outbound packet%s}\n"); p(ips_out_bundlesa, "\t{:send-bundled-sa/%ju} " "{N:/outbound packet%s with bundled SAs}\n"); - p(ips_mbcoalesced, "\t{:mbufs-coalesced-during-clone/%ju} " - "{N:/mbuf%s coalesced during clone}\n"); - p(ips_clcoalesced, "\t{:clusters-coalesced-during-clone/%ju} " - "{N:/cluster%s coalesced during clone}\n"); + p(ips_spdcache_hits, "\t{:spdcache-hits/%ju} " + "{N:/spd cache hit%s}\n"); + p2(ips_spdcache_misses, "\t{:spdcache-misses/%ju} " + "{N:/spd cache miss%s}\n"); p(ips_clcopied, "\t{:clusters-copied-during-clone/%ju} " "{N:/cluster%s copied during clone}\n"); p(ips_mbinserted, "\t{:mbufs-inserted/%ju} " "{N:/mbuf%s inserted during makespace}\n"); +#undef p2 #undef p xo_close_container("ipsec-statistics"); } |