diff options
| author | Sebastian Huber <sebastian.huber@embedded-brains.de> | 2018-08-21 09:39:55 +0200 |
|---|---|---|
| committer | Sebastian Huber <sebastian.huber@embedded-brains.de> | 2018-09-21 10:29:40 +0200 |
| commit | 2df56dbd60bb5d925d2ce0ddbdefdbe6107ea783 (patch) | |
| tree | bd7bad558534db4a1f400bc38a2c9aa7ea4f411e /freebsd/sys | |
| parent | Update to FreeBSD head 2018-02-01 (diff) | |
| download | rtems-libbsd-2df56dbd60bb5d925d2ce0ddbdefdbe6107ea783.tar.bz2 | |
Update to FreeBSD head 2018-04-01
Git mirror commit 8dfb1ccc26d1cea7e2529303003ff61f9f1784c4.
Update #3472.
Diffstat (limited to 'freebsd/sys')
182 files changed, 5483 insertions, 2659 deletions
diff --git a/freebsd/sys/cam/cam_periph.h b/freebsd/sys/cam/cam_periph.h index ee9a5fc0..6eb0084a 100644 --- a/freebsd/sys/cam/cam_periph.h +++ b/freebsd/sys/cam/cam_periph.h @@ -37,6 +37,7 @@ #include <cam/cam_sim.h> #ifdef _KERNEL +#include <sys/sysctl.h> #include <sys/taskqueue.h> #include <cam/cam_xpt.h> @@ -159,7 +160,7 @@ cam_status cam_periph_alloc(periph_ctor_t *periph_ctor, ac_callback_t *, ac_code, void *arg); struct cam_periph *cam_periph_find(struct cam_path *path, char *name); int cam_periph_list(struct cam_path *, struct sbuf *); -cam_status cam_periph_acquire(struct cam_periph *periph); +int cam_periph_acquire(struct cam_periph *periph); void cam_periph_doacquire(struct cam_periph *periph); void cam_periph_release(struct cam_periph *periph); void cam_periph_release_locked(struct cam_periph *periph); @@ -198,6 +199,7 @@ void cam_periph_freeze_after_event(struct cam_periph *periph, u_int duration_ms); int cam_periph_error(union ccb *ccb, cam_flags camflags, u_int32_t sense_flags); +int cam_periph_invalidate_sysctl(SYSCTL_HANDLER_ARGS); static __inline struct mtx * cam_periph_mtx(struct cam_periph *periph) diff --git a/freebsd/sys/cam/scsi/scsi_all.h b/freebsd/sys/cam/scsi/scsi_all.h index 3a11a15e..80f8a221 100644 --- a/freebsd/sys/cam/scsi/scsi_all.h +++ b/freebsd/sys/cam/scsi/scsi_all.h @@ -25,7 +25,11 @@ #define _SCSI_SCSI_ALL_H 1 #include <sys/cdefs.h> +#ifdef _KERNEL #include <machine/stdarg.h> +#else +#include <stdarg.h> +#endif #ifdef _KERNEL /* diff --git a/freebsd/sys/contrib/libb2/blake2-impl.h b/freebsd/sys/contrib/libb2/blake2-impl.h new file mode 100644 index 00000000..222b6127 --- /dev/null +++ b/freebsd/sys/contrib/libb2/blake2-impl.h @@ -0,0 +1,152 @@ +/* + BLAKE2 reference source code package - optimized C implementations + + Written in 2012 by Samuel Neves <sneves@dei.uc.pt> + + To the extent possible under law, the author(s) have dedicated all copyright + and related and neighboring rights to this software to the public domain + worldwide. This software is distributed without any warranty. + + You should have received a copy of the CC0 Public Domain Dedication along with + this software. If not, see <http://creativecommons.org/publicdomain/zero/1.0/>. +*/ +#pragma once +#ifndef __BLAKE2_IMPL_H__ +#define __BLAKE2_IMPL_H__ + +#include <stddef.h> +#include <stdint.h> +#include <string.h> +#include "config.h" + +#define BLAKE2_IMPL_CAT(x,y) x ## y +#define BLAKE2_IMPL_EVAL(x,y) BLAKE2_IMPL_CAT(x,y) +#define BLAKE2_IMPL_NAME(fun) BLAKE2_IMPL_EVAL(fun, SUFFIX) + +static inline uint32_t load32( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) && !defined(HAVE_ALIGNED_ACCESS_REQUIRED) + return *( uint32_t * )( src ); +#else + const uint8_t *p = ( uint8_t * )src; + uint32_t w = *p++; + w |= ( uint32_t )( *p++ ) << 8; + w |= ( uint32_t )( *p++ ) << 16; + w |= ( uint32_t )( *p++ ) << 24; + return w; +#endif +} + +static inline uint64_t load64( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) && !defined(HAVE_ALIGNED_ACCESS_REQUIRED) + return *( uint64_t * )( src ); +#else + const uint8_t *p = ( uint8_t * )src; + uint64_t w = *p++; + w |= ( uint64_t )( *p++ ) << 8; + w |= ( uint64_t )( *p++ ) << 16; + w |= ( uint64_t )( *p++ ) << 24; + w |= ( uint64_t )( *p++ ) << 32; + w |= ( uint64_t )( *p++ ) << 40; + w |= ( uint64_t )( *p++ ) << 48; + w |= ( uint64_t )( *p++ ) << 56; + return w; +#endif +} + +static inline void store32( void *dst, uint32_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) && !defined(HAVE_ALIGNED_ACCESS_REQUIRED) + *( uint32_t * )( dst ) = w; +#else + uint8_t *p = ( uint8_t * )dst; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; +#endif +} + +static inline void store64( void *dst, uint64_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) && !defined(HAVE_ALIGNED_ACCESS_REQUIRED) + *( uint64_t * )( dst ) = w; +#else + uint8_t *p = ( uint8_t * )dst; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; +#endif +} + +static inline uint64_t load48( const void *src ) +{ + const uint8_t *p = ( const uint8_t * )src; + uint64_t w = *p++; + w |= ( uint64_t )( *p++ ) << 8; + w |= ( uint64_t )( *p++ ) << 16; + w |= ( uint64_t )( *p++ ) << 24; + w |= ( uint64_t )( *p++ ) << 32; + w |= ( uint64_t )( *p++ ) << 40; + return w; +} + +static inline void store48( void *dst, uint64_t w ) +{ + uint8_t *p = ( uint8_t * )dst; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; +} + +static inline uint32_t rotl32( const uint32_t w, const unsigned c ) +{ + return ( w << c ) | ( w >> ( 32 - c ) ); +} + +static inline uint64_t rotl64( const uint64_t w, const unsigned c ) +{ + return ( w << c ) | ( w >> ( 64 - c ) ); +} + +static inline uint32_t rotr32( const uint32_t w, const unsigned c ) +{ + return ( w >> c ) | ( w << ( 32 - c ) ); +} + +static inline uint64_t rotr64( const uint64_t w, const unsigned c ) +{ + return ( w >> c ) | ( w << ( 64 - c ) ); +} + +/* prevents compiler optimizing out memset() */ +static inline void secure_zero_memory(void *v, size_t n) +{ +#if defined(_WIN32) || defined(WIN32) + SecureZeroMemory(v, n); +#else +// prioritize first the general C11 call +#if defined(HAVE_MEMSET_S) + memset_s(v, n, 0, n); +#elif defined(HAVE_EXPLICIT_BZERO) + explicit_bzero(v, n); +#elif defined(HAVE_EXPLICIT_MEMSET) + explicit_memset(v, 0, n); +#else + memset(v, 0, n); + __asm__ __volatile__("" :: "r"(v) : "memory"); +#endif +#endif +} + +#endif + diff --git a/freebsd/sys/contrib/libb2/blake2.h b/freebsd/sys/contrib/libb2/blake2.h new file mode 100644 index 00000000..5ca17f61 --- /dev/null +++ b/freebsd/sys/contrib/libb2/blake2.h @@ -0,0 +1,182 @@ +/* + BLAKE2 reference source code package - optimized C implementations + + Written in 2012 by Samuel Neves <sneves@dei.uc.pt> + + To the extent possible under law, the author(s) have dedicated all copyright + and related and neighboring rights to this software to the public domain + worldwide. This software is distributed without any warranty. + + You should have received a copy of the CC0 Public Domain Dedication along with + this software. If not, see <http://creativecommons.org/publicdomain/zero/1.0/>. +*/ +#pragma once +#ifndef __BLAKE2_H__ +#define __BLAKE2_H__ + +#include <stddef.h> +#include <stdint.h> + +#if defined(_WIN32) || defined(__CYGWIN__) + #define BLAKE2_DLL_IMPORT __declspec(dllimport) + #define BLAKE2_DLL_EXPORT __declspec(dllexport) + #define BLAKE2_DLL_PRIVATE +#elif __GNUC__ >= 4 + #define BLAKE2_DLL_IMPORT __attribute__ ((visibility ("default"))) + #define BLAKE2_DLL_EXPORT __attribute__ ((visibility ("default"))) + #define BLAKE2_DLL_PRIVATE __attribute__ ((visibility ("hidden"))) +#else + #define BLAKE2_DLL_IMPORT + #define BLAKE2_DLL_EXPORT + #define BLAKE2_DLL_PRIVATE +#endif + +#if defined(BLAKE2_DLL) + #if defined(BLAKE2_DLL_EXPORTS) // defined if we are building the DLL + #define BLAKE2_API BLAKE2_DLL_EXPORT + #else + #define BLAKE2_API BLAKE2_DLL_IMPORT + #endif + #define BLAKE2_PRIVATE BLAKE2_DLL_PRIVATE // must only be used by hidden logic +#else + #define BLAKE2_API + #define BLAKE2_PRIVATE +#endif + +#if defined(__cplusplus) +extern "C" { +#elif defined(_MSC_VER) && !defined(inline) +#define inline __inline +#endif + + enum blake2s_constant + { + BLAKE2S_BLOCKBYTES = 64, + BLAKE2S_OUTBYTES = 32, + BLAKE2S_KEYBYTES = 32, + BLAKE2S_SALTBYTES = 8, + BLAKE2S_PERSONALBYTES = 8 + }; + + enum blake2b_constant + { + BLAKE2B_BLOCKBYTES = 128, + BLAKE2B_OUTBYTES = 64, + BLAKE2B_KEYBYTES = 64, + BLAKE2B_SALTBYTES = 16, + BLAKE2B_PERSONALBYTES = 16 + }; + +#pragma pack(push, 1) + typedef struct __blake2s_param + { + uint8_t digest_length; // 1 + uint8_t key_length; // 2 + uint8_t fanout; // 3 + uint8_t depth; // 4 + uint32_t leaf_length; // 8 + uint8_t node_offset[6];// 14 + uint8_t node_depth; // 15 + uint8_t inner_length; // 16 + // uint8_t reserved[0]; + uint8_t salt[BLAKE2S_SALTBYTES]; // 24 + uint8_t personal[BLAKE2S_PERSONALBYTES]; // 32 + } blake2s_param; + + typedef struct __blake2s_state + { + uint32_t h[8]; + uint32_t t[2]; + uint32_t f[2]; + uint8_t buf[2 * BLAKE2S_BLOCKBYTES]; + uint32_t buflen; + uint8_t outlen; + uint8_t last_node; + } blake2s_state; + + typedef struct __blake2b_param + { + uint8_t digest_length; // 1 + uint8_t key_length; // 2 + uint8_t fanout; // 3 + uint8_t depth; // 4 + uint32_t leaf_length; // 8 + uint64_t node_offset; // 16 + uint8_t node_depth; // 17 + uint8_t inner_length; // 18 + uint8_t reserved[14]; // 32 + uint8_t salt[BLAKE2B_SALTBYTES]; // 48 + uint8_t personal[BLAKE2B_PERSONALBYTES]; // 64 + } blake2b_param; + + typedef struct __blake2b_state + { + uint64_t h[8]; + uint64_t t[2]; + uint64_t f[2]; + uint8_t buf[2 * BLAKE2B_BLOCKBYTES]; + uint32_t buflen; + uint8_t outlen; + uint8_t last_node; + } blake2b_state; + + typedef struct __blake2sp_state + { + blake2s_state S[8][1]; + blake2s_state R[1]; + uint8_t buf[8 * BLAKE2S_BLOCKBYTES]; + uint32_t buflen; + uint8_t outlen; + } blake2sp_state; + + typedef struct __blake2bp_state + { + blake2b_state S[4][1]; + blake2b_state R[1]; + uint8_t buf[4 * BLAKE2B_BLOCKBYTES]; + uint32_t buflen; + uint8_t outlen; + } blake2bp_state; +#pragma pack(pop) + + // Streaming API + BLAKE2_API int blake2s_init( blake2s_state *S, size_t outlen ); + BLAKE2_API int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ); + BLAKE2_API int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); + BLAKE2_API int blake2s_update( blake2s_state *S, const uint8_t *in, size_t inlen ); + BLAKE2_API int blake2s_final( blake2s_state *S, uint8_t *out, size_t outlen ); + + BLAKE2_API int blake2b_init( blake2b_state *S, size_t outlen ); + BLAKE2_API int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ); + BLAKE2_API int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); + BLAKE2_API int blake2b_update( blake2b_state *S, const uint8_t *in, size_t inlen ); + BLAKE2_API int blake2b_final( blake2b_state *S, uint8_t *out, size_t outlen ); + + BLAKE2_API int blake2sp_init( blake2sp_state *S, size_t outlen ); + BLAKE2_API int blake2sp_init_key( blake2sp_state *S, size_t outlen, const void *key, size_t keylen ); + BLAKE2_API int blake2sp_update( blake2sp_state *S, const uint8_t *in, size_t inlen ); + BLAKE2_API int blake2sp_final( blake2sp_state *S, uint8_t *out, size_t outlen ); + + BLAKE2_API int blake2bp_init( blake2bp_state *S, size_t outlen ); + BLAKE2_API int blake2bp_init_key( blake2bp_state *S, size_t outlen, const void *key, size_t keylen ); + BLAKE2_API int blake2bp_update( blake2bp_state *S, const uint8_t *in, size_t inlen ); + BLAKE2_API int blake2bp_final( blake2bp_state *S, uint8_t *out, size_t outlen ); + + // Simple API + BLAKE2_API int blake2s( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); + BLAKE2_API int blake2b( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); + + BLAKE2_API int blake2sp( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); + BLAKE2_API int blake2bp( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); + + static inline int blake2( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ) + { + return blake2b( out, in, key, outlen, inlen, keylen ); + } + +#if defined(__cplusplus) +} +#endif + +#endif + diff --git a/freebsd/sys/contrib/libb2/blake2b-ref.c b/freebsd/sys/contrib/libb2/blake2b-ref.c new file mode 100644 index 00000000..152bd9a5 --- /dev/null +++ b/freebsd/sys/contrib/libb2/blake2b-ref.c @@ -0,0 +1,391 @@ +#include <machine/rtems-bsd-kernel-space.h> + +/* + BLAKE2 reference source code package - reference C implementations + + Written in 2012 by Samuel Neves <sneves@dei.uc.pt> + + To the extent possible under law, the author(s) have dedicated all copyright + and related and neighboring rights to this software to the public domain + worldwide. This software is distributed without any warranty. + + You should have received a copy of the CC0 Public Domain Dedication along with + this software. If not, see <http://creativecommons.org/publicdomain/zero/1.0/>. +*/ + +#include <stdint.h> +#include <string.h> +#include <stdio.h> + +#include "blake2.h" +#ifdef __rtems__ +#define SUFFIX _ref +#endif /* __rtems__ */ +#include "blake2-impl.h" + +static const uint64_t blake2b_IV[8] = +{ + 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, + 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, + 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, + 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL +}; + +static const uint8_t blake2b_sigma[12][16] = +{ + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } +}; + + +static inline int blake2b_set_lastnode( blake2b_state *S ) +{ + S->f[1] = ~0ULL; + return 0; +} + +static inline int blake2b_clear_lastnode( blake2b_state *S ) +{ + S->f[1] = 0ULL; + return 0; +} + +/* Some helper functions, not necessarily useful */ +static inline int blake2b_set_lastblock( blake2b_state *S ) +{ + if( S->last_node ) blake2b_set_lastnode( S ); + + S->f[0] = ~0ULL; + return 0; +} + +static inline int blake2b_clear_lastblock( blake2b_state *S ) +{ + if( S->last_node ) blake2b_clear_lastnode( S ); + + S->f[0] = 0ULL; + return 0; +} + +static inline int blake2b_increment_counter( blake2b_state *S, const uint64_t inc ) +{ + S->t[0] += inc; + S->t[1] += ( S->t[0] < inc ); + return 0; +} + + + +// Parameter-related functions +static inline int blake2b_param_set_digest_length( blake2b_param *P, const uint8_t digest_length ) +{ + P->digest_length = digest_length; + return 0; +} + +static inline int blake2b_param_set_fanout( blake2b_param *P, const uint8_t fanout ) +{ + P->fanout = fanout; + return 0; +} + +static inline int blake2b_param_set_max_depth( blake2b_param *P, const uint8_t depth ) +{ + P->depth = depth; + return 0; +} + +static inline int blake2b_param_set_leaf_length( blake2b_param *P, const uint32_t leaf_length ) +{ + store32( &P->leaf_length, leaf_length ); + return 0; +} + +static inline int blake2b_param_set_node_offset( blake2b_param *P, const uint64_t node_offset ) +{ + store64( &P->node_offset, node_offset ); + return 0; +} + +static inline int blake2b_param_set_node_depth( blake2b_param *P, const uint8_t node_depth ) +{ + P->node_depth = node_depth; + return 0; +} + +static inline int blake2b_param_set_inner_length( blake2b_param *P, const uint8_t inner_length ) +{ + P->inner_length = inner_length; + return 0; +} + +static inline int blake2b_param_set_salt( blake2b_param *P, const uint8_t salt[BLAKE2B_SALTBYTES] ) +{ + memcpy( P->salt, salt, BLAKE2B_SALTBYTES ); + return 0; +} + +static inline int blake2b_param_set_personal( blake2b_param *P, const uint8_t personal[BLAKE2B_PERSONALBYTES] ) +{ + memcpy( P->personal, personal, BLAKE2B_PERSONALBYTES ); + return 0; +} + +static inline int blake2b_init0( blake2b_state *S ) +{ + memset( S, 0, sizeof( blake2b_state ) ); + + for( int i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i]; + + return 0; +} + +#define blake2b_init BLAKE2_IMPL_NAME(blake2b_init) +#define blake2b_init_param BLAKE2_IMPL_NAME(blake2b_init_param) +#define blake2b_init_key BLAKE2_IMPL_NAME(blake2b_init_key) +#define blake2b_update BLAKE2_IMPL_NAME(blake2b_update) +#define blake2b_final BLAKE2_IMPL_NAME(blake2b_final) +#define blake2b BLAKE2_IMPL_NAME(blake2b) + +#if defined(__cplusplus) +extern "C" { +#endif + int blake2b_init( blake2b_state *S, size_t outlen ); + int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); + int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2b_update( blake2b_state *S, const uint8_t *in, size_t inlen ); + int blake2b_final( blake2b_state *S, uint8_t *out, size_t outlen ); + int blake2b( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); +#if defined(__cplusplus) +} +#endif + +/* init xors IV with input parameter block */ +int blake2b_init_param( blake2b_state *S, const blake2b_param *P ) +{ + blake2b_init0( S ); + uint8_t *p = ( uint8_t * )( P ); + + /* IV XOR ParamBlock */ + for( size_t i = 0; i < 8; ++i ) + S->h[i] ^= load64( p + sizeof( S->h[i] ) * i ); + + S->outlen = P->digest_length; + return 0; +} + + + +int blake2b_init( blake2b_state *S, size_t outlen ) +{ + blake2b_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + P->digest_length = ( uint8_t ) outlen; + P->key_length = 0; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store64( &P->node_offset, 0 ); + P->node_depth = 0; + P->inner_length = 0; + memset( P->reserved, 0, sizeof( P->reserved ) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + return blake2b_init_param( S, P ); +} + + +int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ) +{ + blake2b_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + if ( !key || !keylen || keylen > BLAKE2B_KEYBYTES ) return -1; + + P->digest_length = ( uint8_t ) outlen; + P->key_length = ( uint8_t ) keylen; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store64( &P->node_offset, 0 ); + P->node_depth = 0; + P->inner_length = 0; + memset( P->reserved, 0, sizeof( P->reserved ) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + + if( blake2b_init_param( S, P ) < 0 ) return -1; + + { + uint8_t block[BLAKE2B_BLOCKBYTES]; + memset( block, 0, BLAKE2B_BLOCKBYTES ); + memcpy( block, key, keylen ); + blake2b_update( S, block, BLAKE2B_BLOCKBYTES ); + secure_zero_memory( block, BLAKE2B_BLOCKBYTES ); /* Burn the key from stack */ + } + return 0; +} + +static int blake2b_compress( blake2b_state *S, const uint8_t block[BLAKE2B_BLOCKBYTES] ) +{ + uint64_t m[16]; + uint64_t v[16]; + size_t i; + + for( i = 0; i < 16; ++i ) + m[i] = load64( block + i * sizeof( m[i] ) ); + + for( i = 0; i < 8; ++i ) + v[i] = S->h[i]; + + v[ 8] = blake2b_IV[0]; + v[ 9] = blake2b_IV[1]; + v[10] = blake2b_IV[2]; + v[11] = blake2b_IV[3]; + v[12] = S->t[0] ^ blake2b_IV[4]; + v[13] = S->t[1] ^ blake2b_IV[5]; + v[14] = S->f[0] ^ blake2b_IV[6]; + v[15] = S->f[1] ^ blake2b_IV[7]; +#define G(r,i,a,b,c,d) \ + do { \ + a = a + b + m[blake2b_sigma[r][2*i+0]]; \ + d = rotr64(d ^ a, 32); \ + c = c + d; \ + b = rotr64(b ^ c, 24); \ + a = a + b + m[blake2b_sigma[r][2*i+1]]; \ + d = rotr64(d ^ a, 16); \ + c = c + d; \ + b = rotr64(b ^ c, 63); \ + } while(0) +#define ROUND(r) \ + do { \ + G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ + G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ + G(r,2,v[ 2],v[ 6],v[10],v[14]); \ + G(r,3,v[ 3],v[ 7],v[11],v[15]); \ + G(r,4,v[ 0],v[ 5],v[10],v[15]); \ + G(r,5,v[ 1],v[ 6],v[11],v[12]); \ + G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ + G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ + } while(0) + ROUND( 0 ); + ROUND( 1 ); + ROUND( 2 ); + ROUND( 3 ); + ROUND( 4 ); + ROUND( 5 ); + ROUND( 6 ); + ROUND( 7 ); + ROUND( 8 ); + ROUND( 9 ); + ROUND( 10 ); + ROUND( 11 ); + + for( i = 0; i < 8; ++i ) + S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; + +#undef G +#undef ROUND + return 0; +} + + +int blake2b_update( blake2b_state *S, const uint8_t *in, size_t inlen ) +{ + while( inlen > 0 ) + { + uint32_t left = S->buflen; + uint32_t fill = 2 * BLAKE2B_BLOCKBYTES - left; + + if( inlen > fill ) + { + memcpy( S->buf + left, in, fill ); // Fill buffer + S->buflen += fill; + blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); + blake2b_compress( S, S->buf ); // Compress + memcpy( S->buf, S->buf + BLAKE2B_BLOCKBYTES, BLAKE2B_BLOCKBYTES ); // Shift buffer left + S->buflen -= BLAKE2B_BLOCKBYTES; + in += fill; + inlen -= fill; + } + else // inlen <= fill + { + memcpy( S->buf + left, in, inlen ); + S->buflen += ( uint32_t ) inlen; // Be lazy, do not compress + in += inlen; + inlen -= inlen; + } + } + + return 0; +} + +int blake2b_final( blake2b_state *S, uint8_t *out, size_t outlen ) +{ + uint8_t buffer[BLAKE2B_OUTBYTES]; + size_t i; + + if(S->outlen != outlen) return -1; + + if( S->buflen > BLAKE2B_BLOCKBYTES ) + { + blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); + blake2b_compress( S, S->buf ); + S->buflen -= BLAKE2B_BLOCKBYTES; + memcpy( S->buf, S->buf + BLAKE2B_BLOCKBYTES, S->buflen ); + } + + blake2b_increment_counter( S, S->buflen ); + blake2b_set_lastblock( S ); + memset( S->buf + S->buflen, 0, 2 * BLAKE2B_BLOCKBYTES - S->buflen ); /* Padding */ + blake2b_compress( S, S->buf ); + + for( i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ + store64( buffer + sizeof( S->h[i] ) * i, S->h[i] ); + + memcpy( out, buffer, outlen ); + return 0; +} + +int blake2b( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ) +{ + blake2b_state S[1]; + + /* Verify parameters */ + if ( NULL == in && inlen > 0 ) return -1; + + if ( NULL == out ) return -1; + + if( NULL == key && keylen > 0 ) return -1; + + if( !outlen || outlen > BLAKE2B_OUTBYTES ) return -1; + + if( keylen > BLAKE2B_KEYBYTES ) return -1; + + if( keylen > 0 ) + { + if( blake2b_init_key( S, outlen, key, keylen ) < 0 ) return -1; + } + else + { + if( blake2b_init( S, outlen ) < 0 ) return -1; + } + + if( blake2b_update( S, ( uint8_t * )in, inlen ) < 0 ) return -1; + return blake2b_final( S, out, outlen ); +} + + diff --git a/freebsd/sys/contrib/libb2/blake2s-ref.c b/freebsd/sys/contrib/libb2/blake2s-ref.c new file mode 100644 index 00000000..44449736 --- /dev/null +++ b/freebsd/sys/contrib/libb2/blake2s-ref.c @@ -0,0 +1,380 @@ +#include <machine/rtems-bsd-kernel-space.h> + +/* + BLAKE2 reference source code package - reference C implementations + + Written in 2012 by Samuel Neves <sneves@dei.uc.pt> + + To the extent possible under law, the author(s) have dedicated all copyright + and related and neighboring rights to this software to the public domain + worldwide. This software is distributed without any warranty. + + You should have received a copy of the CC0 Public Domain Dedication along with + this software. If not, see <http://creativecommons.org/publicdomain/zero/1.0/>. +*/ + +#include <stdint.h> +#include <string.h> +#include <stdio.h> + +#include "blake2.h" +#ifdef __rtems__ +#define SUFFIX _ref +#endif /* __rtems__ */ +#include "blake2-impl.h" + +static const uint32_t blake2s_IV[8] = +{ + 0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 0xA54FF53AUL, + 0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL +}; + +static const uint8_t blake2s_sigma[10][16] = +{ + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , +}; + +static inline int blake2s_set_lastnode( blake2s_state *S ) +{ + S->f[1] = ~0U; + return 0; +} + +static inline int blake2s_clear_lastnode( blake2s_state *S ) +{ + S->f[1] = 0U; + return 0; +} + +/* Some helper functions, not necessarily useful */ +static inline int blake2s_set_lastblock( blake2s_state *S ) +{ + if( S->last_node ) blake2s_set_lastnode( S ); + + S->f[0] = ~0U; + return 0; +} + +static inline int blake2s_clear_lastblock( blake2s_state *S ) +{ + if( S->last_node ) blake2s_clear_lastnode( S ); + + S->f[0] = 0U; + return 0; +} + +static inline int blake2s_increment_counter( blake2s_state *S, const uint32_t inc ) +{ + S->t[0] += inc; + S->t[1] += ( S->t[0] < inc ); + return 0; +} + +// Parameter-related functions +static inline int blake2s_param_set_digest_length( blake2s_param *P, const uint8_t digest_length ) +{ + P->digest_length = digest_length; + return 0; +} + +static inline int blake2s_param_set_fanout( blake2s_param *P, const uint8_t fanout ) +{ + P->fanout = fanout; + return 0; +} + +static inline int blake2s_param_set_max_depth( blake2s_param *P, const uint8_t depth ) +{ + P->depth = depth; + return 0; +} + +static inline int blake2s_param_set_leaf_length( blake2s_param *P, const uint32_t leaf_length ) +{ + store32( &P->leaf_length, leaf_length ); + return 0; +} + +static inline int blake2s_param_set_node_offset( blake2s_param *P, const uint64_t node_offset ) +{ + store48( P->node_offset, node_offset ); + return 0; +} + +static inline int blake2s_param_set_node_depth( blake2s_param *P, const uint8_t node_depth ) +{ + P->node_depth = node_depth; + return 0; +} + +static inline int blake2s_param_set_inner_length( blake2s_param *P, const uint8_t inner_length ) +{ + P->inner_length = inner_length; + return 0; +} + +static inline int blake2s_param_set_salt( blake2s_param *P, const uint8_t salt[BLAKE2S_SALTBYTES] ) +{ + memcpy( P->salt, salt, BLAKE2S_SALTBYTES ); + return 0; +} + +static inline int blake2s_param_set_personal( blake2s_param *P, const uint8_t personal[BLAKE2S_PERSONALBYTES] ) +{ + memcpy( P->personal, personal, BLAKE2S_PERSONALBYTES ); + return 0; +} + +static inline int blake2s_init0( blake2s_state *S ) +{ + memset( S, 0, sizeof( blake2s_state ) ); + + for( int i = 0; i < 8; ++i ) S->h[i] = blake2s_IV[i]; + + return 0; +} + +#define blake2s_init BLAKE2_IMPL_NAME(blake2s_init) +#define blake2s_init_param BLAKE2_IMPL_NAME(blake2s_init_param) +#define blake2s_init_key BLAKE2_IMPL_NAME(blake2s_init_key) +#define blake2s_update BLAKE2_IMPL_NAME(blake2s_update) +#define blake2s_final BLAKE2_IMPL_NAME(blake2s_final) +#define blake2s BLAKE2_IMPL_NAME(blake2s) + +#if defined(__cplusplus) +extern "C" { +#endif + int blake2s_init( blake2s_state *S, size_t outlen ); + int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); + int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2s_update( blake2s_state *S, const uint8_t *in, size_t inlen ); + int blake2s_final( blake2s_state *S, uint8_t *out, size_t outlen ); + int blake2s( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); +#if defined(__cplusplus) +} +#endif + +/* init2 xors IV with input parameter block */ +int blake2s_init_param( blake2s_state *S, const blake2s_param *P ) +{ + blake2s_init0( S ); + uint32_t *p = ( uint32_t * )( P ); + + /* IV XOR ParamBlock */ + for( size_t i = 0; i < 8; ++i ) + S->h[i] ^= load32( &p[i] ); + + S->outlen = P->digest_length; + return 0; +} + + +// Sequential blake2s initialization +int blake2s_init( blake2s_state *S, size_t outlen ) +{ + blake2s_param P[1]; + + /* Move interval verification here? */ + if ( ( !outlen ) || ( outlen > BLAKE2S_OUTBYTES ) ) return -1; + + P->digest_length = ( uint8_t) outlen; + P->key_length = 0; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store48( &P->node_offset, 0 ); + P->node_depth = 0; + P->inner_length = 0; + // memset(P->reserved, 0, sizeof(P->reserved) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + return blake2s_init_param( S, P ); +} + +int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ) +{ + blake2s_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2S_OUTBYTES ) ) return -1; + + if ( !key || !keylen || keylen > BLAKE2S_KEYBYTES ) return -1; + + P->digest_length = ( uint8_t ) outlen; + P->key_length = ( uint8_t ) keylen; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store48( &P->node_offset, 0 ); + P->node_depth = 0; + P->inner_length = 0; + // memset(P->reserved, 0, sizeof(P->reserved) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + + if( blake2s_init_param( S, P ) < 0 ) return -1; + + { + uint8_t block[BLAKE2S_BLOCKBYTES]; + memset( block, 0, BLAKE2S_BLOCKBYTES ); + memcpy( block, key, keylen ); + blake2s_update( S, block, BLAKE2S_BLOCKBYTES ); + secure_zero_memory( block, BLAKE2S_BLOCKBYTES ); /* Burn the key from stack */ + } + return 0; +} + +static int blake2s_compress( blake2s_state *S, const uint8_t block[BLAKE2S_BLOCKBYTES] ) +{ + uint32_t m[16]; + uint32_t v[16]; + + for( size_t i = 0; i < 16; ++i ) + m[i] = load32( block + i * sizeof( m[i] ) ); + + for( size_t i = 0; i < 8; ++i ) + v[i] = S->h[i]; + + v[ 8] = blake2s_IV[0]; + v[ 9] = blake2s_IV[1]; + v[10] = blake2s_IV[2]; + v[11] = blake2s_IV[3]; + v[12] = S->t[0] ^ blake2s_IV[4]; + v[13] = S->t[1] ^ blake2s_IV[5]; + v[14] = S->f[0] ^ blake2s_IV[6]; + v[15] = S->f[1] ^ blake2s_IV[7]; +#define G(r,i,a,b,c,d) \ + do { \ + a = a + b + m[blake2s_sigma[r][2*i+0]]; \ + d = rotr32(d ^ a, 16); \ + c = c + d; \ + b = rotr32(b ^ c, 12); \ + a = a + b + m[blake2s_sigma[r][2*i+1]]; \ + d = rotr32(d ^ a, 8); \ + c = c + d; \ + b = rotr32(b ^ c, 7); \ + } while(0) +#define ROUND(r) \ + do { \ + G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ + G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ + G(r,2,v[ 2],v[ 6],v[10],v[14]); \ + G(r,3,v[ 3],v[ 7],v[11],v[15]); \ + G(r,4,v[ 0],v[ 5],v[10],v[15]); \ + G(r,5,v[ 1],v[ 6],v[11],v[12]); \ + G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ + G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ + } while(0) + ROUND( 0 ); + ROUND( 1 ); + ROUND( 2 ); + ROUND( 3 ); + ROUND( 4 ); + ROUND( 5 ); + ROUND( 6 ); + ROUND( 7 ); + ROUND( 8 ); + ROUND( 9 ); + + for( size_t i = 0; i < 8; ++i ) + S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; + +#undef G +#undef ROUND + return 0; +} + + +int blake2s_update( blake2s_state *S, const uint8_t *in, size_t inlen ) +{ + while( inlen > 0 ) + { + uint32_t left = S->buflen; + uint32_t fill = 2 * BLAKE2S_BLOCKBYTES - left; + + if( inlen > fill ) + { + memcpy( S->buf + left, in, fill ); // Fill buffer + S->buflen += fill; + blake2s_increment_counter( S, BLAKE2S_BLOCKBYTES ); + blake2s_compress( S, S->buf ); // Compress + memcpy( S->buf, S->buf + BLAKE2S_BLOCKBYTES, BLAKE2S_BLOCKBYTES ); // Shift buffer left + S->buflen -= BLAKE2S_BLOCKBYTES; + in += fill; + inlen -= fill; + } + else // inlen <= fill + { + memcpy( S->buf + left, in, inlen ); + S->buflen += ( uint32_t ) inlen; // Be lazy, do not compress + in += inlen; + inlen -= inlen; + } + } + + return 0; +} + +int blake2s_final( blake2s_state *S, uint8_t *out, size_t outlen ) +{ + uint8_t buffer[BLAKE2S_OUTBYTES]; + size_t i; + + if(S->outlen != outlen) return -1; + + if( S->buflen > BLAKE2S_BLOCKBYTES ) + { + blake2s_increment_counter( S, BLAKE2S_BLOCKBYTES ); + blake2s_compress( S, S->buf ); + S->buflen -= BLAKE2S_BLOCKBYTES; + memcpy( S->buf, S->buf + BLAKE2S_BLOCKBYTES, S->buflen ); + } + + blake2s_increment_counter( S, ( uint32_t )S->buflen ); + blake2s_set_lastblock( S ); + memset( S->buf + S->buflen, 0, 2 * BLAKE2S_BLOCKBYTES - S->buflen ); /* Padding */ + blake2s_compress( S, S->buf ); + + for( i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ + store32( buffer + sizeof( S->h[i] ) * i, S->h[i] ); + + memcpy( out, buffer, outlen ); + return 0; +} + +int blake2s( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ) +{ + blake2s_state S[1]; + + /* Verify parameters */ + if ( NULL == in && inlen > 0 ) return -1; + + if ( NULL == out ) return -1; + + if ( NULL == key && keylen > 0 ) return -1; + + if( !outlen || outlen > BLAKE2S_OUTBYTES ) return -1; + + if( keylen > BLAKE2S_KEYBYTES ) return -1; + + if( keylen > 0 ) + { + if( blake2s_init_key( S, outlen, key, keylen ) < 0 ) return -1; + } + else + { + if( blake2s_init( S, outlen ) < 0 ) return -1; + } + + if( blake2s_update( S, ( uint8_t * )in, inlen ) < 0) return -1; + return blake2s_final( S, out, outlen ); +} + diff --git a/freebsd/sys/crypto/blake2/blake2-sw.c b/freebsd/sys/crypto/blake2/blake2-sw.c new file mode 100644 index 00000000..807c1ea9 --- /dev/null +++ b/freebsd/sys/crypto/blake2/blake2-sw.c @@ -0,0 +1,165 @@ +#include <machine/rtems-bsd-kernel-space.h> + +/* This file is in the public domain. */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <contrib/libb2/blake2.h> +#include <opencrypto/xform_auth.h> + +extern int blake2b_init_ref(blake2b_state *S, size_t outlen); +extern int blake2b_init_param_ref(blake2b_state *S, const blake2b_param *P); +extern int blake2b_init_key_ref(blake2b_state *S, size_t outlen, + const void *key, size_t keylen); +extern int blake2b_update_ref(blake2b_state *S, const uint8_t *in, + size_t inlen); +extern int blake2b_final_ref(blake2b_state *S, uint8_t *out, size_t outlen); +extern int blake2b_ref(uint8_t *out, const void *in, const void *key, + size_t outlen, size_t inlen, size_t keylen); + +extern int blake2s_init_ref(blake2s_state *S, size_t outlen); +extern int blake2s_init_param_ref(blake2s_state *S, const blake2s_param *P); +extern int blake2s_init_key_ref(blake2s_state *S, size_t outlen, + const void *key, size_t keylen); +extern int blake2s_update_ref(blake2s_state *S, const uint8_t *in, + size_t inlen); +extern int blake2s_final_ref(blake2s_state *S, uint8_t *out, size_t outlen); +extern int blake2s_ref(uint8_t *out, const void *in, const void *key, + size_t outlen, size_t inlen, size_t keylen); + +struct blake2b_xform_ctx { + blake2b_state state; + uint8_t key[BLAKE2B_KEYBYTES]; + uint16_t klen; +}; +CTASSERT(sizeof(union authctx) >= sizeof(struct blake2b_xform_ctx)); + +static void +blake2b_xform_init(void *vctx) +{ + struct blake2b_xform_ctx *ctx = vctx; + int rc; + + if (ctx->klen > 0) + rc = blake2b_init_key_ref(&ctx->state, BLAKE2B_OUTBYTES, + ctx->key, ctx->klen); + else + rc = blake2b_init_ref(&ctx->state, BLAKE2B_OUTBYTES); + if (rc != 0) + panic("blake2b_init_key: invalid arguments"); +} + +static void +blake2b_xform_setkey(void *vctx, const uint8_t *key, uint16_t klen) +{ + struct blake2b_xform_ctx *ctx = vctx; + + if (klen > sizeof(ctx->key)) + panic("invalid klen %u", (unsigned)klen); + memcpy(ctx->key, key, klen); + ctx->klen = klen; +} + +static int +blake2b_xform_update(void *vctx, const uint8_t *data, uint16_t len) +{ + struct blake2b_xform_ctx *ctx = vctx; + int rc; + + rc = blake2b_update_ref(&ctx->state, data, len); + if (rc != 0) + return (EINVAL); + return (0); +} + +static void +blake2b_xform_final(uint8_t *out, void *vctx) +{ + struct blake2b_xform_ctx *ctx = vctx; + int rc; + + rc = blake2b_final_ref(&ctx->state, out, BLAKE2B_OUTBYTES); + if (rc != 0) + panic("blake2b_final: invalid"); +} + +struct auth_hash auth_hash_blake2b = { + .type = CRYPTO_BLAKE2B, + .name = "Blake2b", + .keysize = BLAKE2B_KEYBYTES, + .hashsize = BLAKE2B_OUTBYTES, + .ctxsize = sizeof(struct blake2b_xform_ctx), + .Setkey = blake2b_xform_setkey, + .Init = blake2b_xform_init, + .Update = blake2b_xform_update, + .Final = blake2b_xform_final, +}; + +struct blake2s_xform_ctx { + blake2s_state state; + uint8_t key[BLAKE2S_KEYBYTES]; + uint16_t klen; +}; +CTASSERT(sizeof(union authctx) >= sizeof(struct blake2s_xform_ctx)); + +static void +blake2s_xform_init(void *vctx) +{ + struct blake2s_xform_ctx *ctx = vctx; + int rc; + + if (ctx->klen > 0) + rc = blake2s_init_key_ref(&ctx->state, BLAKE2S_OUTBYTES, + ctx->key, ctx->klen); + else + rc = blake2s_init_ref(&ctx->state, BLAKE2S_OUTBYTES); + if (rc != 0) + panic("blake2s_init_key: invalid arguments"); +} + +static void +blake2s_xform_setkey(void *vctx, const uint8_t *key, uint16_t klen) +{ + struct blake2s_xform_ctx *ctx = vctx; + + if (klen > sizeof(ctx->key)) + panic("invalid klen %u", (unsigned)klen); + memcpy(ctx->key, key, klen); + ctx->klen = klen; +} + +static int +blake2s_xform_update(void *vctx, const uint8_t *data, uint16_t len) +{ + struct blake2s_xform_ctx *ctx = vctx; + int rc; + + rc = blake2s_update_ref(&ctx->state, data, len); + if (rc != 0) + return (EINVAL); + return (0); +} + +static void +blake2s_xform_final(uint8_t *out, void *vctx) +{ + struct blake2s_xform_ctx *ctx = vctx; + int rc; + + rc = blake2s_final_ref(&ctx->state, out, BLAKE2S_OUTBYTES); + if (rc != 0) + panic("blake2s_final: invalid"); +} + +struct auth_hash auth_hash_blake2s = { + .type = CRYPTO_BLAKE2S, + .name = "Blake2s", + .keysize = BLAKE2S_KEYBYTES, + .hashsize = BLAKE2S_OUTBYTES, + .ctxsize = sizeof(struct blake2s_xform_ctx), + .Setkey = blake2s_xform_setkey, + .Init = blake2s_xform_init, + .Update = blake2s_xform_update, + .Final = blake2s_xform_final, +}; diff --git a/freebsd/sys/crypto/chacha20/chacha-sw.c b/freebsd/sys/crypto/chacha20/chacha-sw.c new file mode 100644 index 00000000..0a03d91b --- /dev/null +++ b/freebsd/sys/crypto/chacha20/chacha-sw.c @@ -0,0 +1,80 @@ +#include <machine/rtems-bsd-kernel-space.h> + +/* This file is in the public domain. */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <crypto/chacha20/chacha.h> +#include <opencrypto/xform_enc.h> + +static int +chacha20_xform_setkey(u_int8_t **sched, u_int8_t *key, int len) +{ + struct chacha_ctx *ctx; + + if (len != CHACHA_MINKEYLEN && len != 32) + return (EINVAL); + + ctx = malloc(sizeof(*ctx), M_CRYPTO_DATA, M_NOWAIT | M_ZERO); + *sched = (void *)ctx; + if (ctx == NULL) + return (ENOMEM); + + chacha_keysetup(ctx, key, len * 8); + return (0); +} + +static void +chacha20_xform_reinit(caddr_t key, u_int8_t *iv) +{ + struct chacha_ctx *ctx; + + ctx = (void *)key; + chacha_ivsetup(ctx, iv + 8, iv); +} + +static void +chacha20_xform_zerokey(u_int8_t **sched) +{ + struct chacha_ctx *ctx; + + ctx = (void *)*sched; + explicit_bzero(ctx, sizeof(*ctx)); + free(ctx, M_CRYPTO_DATA); + *sched = NULL; +} + +static void +chacha20_xform_crypt(caddr_t cctx, u_int8_t *bytes) +{ + struct chacha_ctx *ctx; + + ctx = (void *)cctx; + chacha_encrypt_bytes(ctx, bytes, bytes, 1); +} + +static void +chacha20_xform_crypt_multi(void *vctx, uint8_t *bytes, size_t len) +{ + struct chacha_ctx *ctx; + + ctx = vctx; + chacha_encrypt_bytes(ctx, bytes, bytes, len); +} + +struct enc_xform enc_xform_chacha20 = { + .type = CRYPTO_CHACHA20, + .name = "chacha20", + .blocksize = 1, + .ivsize = CHACHA_NONCELEN + CHACHA_CTRLEN, + .minkey = CHACHA_MINKEYLEN, + .maxkey = 32, + .encrypt = chacha20_xform_crypt, + .decrypt = chacha20_xform_crypt, + .setkey = chacha20_xform_setkey, + .zerokey = chacha20_xform_zerokey, + .reinit = chacha20_xform_reinit, + .encrypt_multi = chacha20_xform_crypt_multi, + .decrypt_multi = chacha20_xform_crypt_multi, +}; diff --git a/freebsd/sys/crypto/chacha20/chacha.c b/freebsd/sys/crypto/chacha20/chacha.c new file mode 100644 index 00000000..74d7fe02 --- /dev/null +++ b/freebsd/sys/crypto/chacha20/chacha.c @@ -0,0 +1,234 @@ +#include <machine/rtems-bsd-kernel-space.h> + +/* +chacha-merged.c version 20080118 +D. J. Bernstein +Public domain. +*/ + +/* $OpenBSD: chacha.c,v 1.1 2013/11/21 00:45:44 djm Exp $ */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/types.h> + +#include <crypto/chacha20/chacha.h> + + +typedef uint8_t u8; +typedef uint32_t u32; + +typedef struct chacha_ctx chacha_ctx; + +#define U8C(v) (v##U) +#define U32C(v) (v##U) + +#define U8V(v) ((u8)(v) & U8C(0xFF)) +#define U32V(v) ((u32)(v) & U32C(0xFFFFFFFF)) + +#define ROTL32(v, n) \ + (U32V((v) << (n)) | ((v) >> (32 - (n)))) + +#define U8TO32_LITTLE(p) \ + (((u32)((p)[0]) ) | \ + ((u32)((p)[1]) << 8) | \ + ((u32)((p)[2]) << 16) | \ + ((u32)((p)[3]) << 24)) + +#define U32TO8_LITTLE(p, v) \ + do { \ + (p)[0] = U8V((v) ); \ + (p)[1] = U8V((v) >> 8); \ + (p)[2] = U8V((v) >> 16); \ + (p)[3] = U8V((v) >> 24); \ + } while (0) + +#define ROTATE(v,c) (ROTL32(v,c)) +#define XOR(v,w) ((v) ^ (w)) +#define PLUS(v,w) (U32V((v) + (w))) +#define PLUSONE(v) (PLUS((v),1)) + +#define QUARTERROUND(a,b,c,d) \ + a = PLUS(a,b); d = ROTATE(XOR(d,a),16); \ + c = PLUS(c,d); b = ROTATE(XOR(b,c),12); \ + a = PLUS(a,b); d = ROTATE(XOR(d,a), 8); \ + c = PLUS(c,d); b = ROTATE(XOR(b,c), 7); + +static const char sigma[16] = "expand 32-byte k"; +static const char tau[16] = "expand 16-byte k"; + +void +#ifndef __rtems__ +chacha_keysetup(chacha_ctx *x,const u8 *k,u32 kbits) +#else /* __rtems__ */ +chacha_keysetup(chacha_ctx *x,const u8 *k,u_int kbits) +#endif /* __rtems__ */ +{ + const char *constants; + + x->input[4] = U8TO32_LITTLE(k + 0); + x->input[5] = U8TO32_LITTLE(k + 4); + x->input[6] = U8TO32_LITTLE(k + 8); + x->input[7] = U8TO32_LITTLE(k + 12); + if (kbits == 256) { /* recommended */ + k += 16; + constants = sigma; + } else { /* kbits == 128 */ + constants = tau; + } + x->input[8] = U8TO32_LITTLE(k + 0); + x->input[9] = U8TO32_LITTLE(k + 4); + x->input[10] = U8TO32_LITTLE(k + 8); + x->input[11] = U8TO32_LITTLE(k + 12); + x->input[0] = U8TO32_LITTLE(constants + 0); + x->input[1] = U8TO32_LITTLE(constants + 4); + x->input[2] = U8TO32_LITTLE(constants + 8); + x->input[3] = U8TO32_LITTLE(constants + 12); +} + +void +chacha_ivsetup(chacha_ctx *x, const u8 *iv, const u8 *counter) +{ + x->input[12] = counter == NULL ? 0 : U8TO32_LITTLE(counter + 0); + x->input[13] = counter == NULL ? 0 : U8TO32_LITTLE(counter + 4); + x->input[14] = U8TO32_LITTLE(iv + 0); + x->input[15] = U8TO32_LITTLE(iv + 4); +} + +void +#ifndef __rtems__ +chacha_encrypt_bytes(chacha_ctx *x,const u8 *m,u8 *c,u32 bytes) +#else /* __rtems__ */ +chacha_encrypt_bytes(chacha_ctx *x,const u8 *m,u8 *c,u_int bytes) +#endif /* __rtems__ */ +{ + u32 x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15; + u32 j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15; + u8 *ctarget = NULL; + u8 tmp[64]; + u_int i; + + if (!bytes) return; + + j0 = x->input[0]; + j1 = x->input[1]; + j2 = x->input[2]; + j3 = x->input[3]; + j4 = x->input[4]; + j5 = x->input[5]; + j6 = x->input[6]; + j7 = x->input[7]; + j8 = x->input[8]; + j9 = x->input[9]; + j10 = x->input[10]; + j11 = x->input[11]; + j12 = x->input[12]; + j13 = x->input[13]; + j14 = x->input[14]; + j15 = x->input[15]; + + for (;;) { + if (bytes < 64) { + for (i = 0;i < bytes;++i) tmp[i] = m[i]; + m = tmp; + ctarget = c; + c = tmp; + } + x0 = j0; + x1 = j1; + x2 = j2; + x3 = j3; + x4 = j4; + x5 = j5; + x6 = j6; + x7 = j7; + x8 = j8; + x9 = j9; + x10 = j10; + x11 = j11; + x12 = j12; + x13 = j13; + x14 = j14; + x15 = j15; + for (i = 20;i > 0;i -= 2) { + QUARTERROUND( x0, x4, x8,x12) + QUARTERROUND( x1, x5, x9,x13) + QUARTERROUND( x2, x6,x10,x14) + QUARTERROUND( x3, x7,x11,x15) + QUARTERROUND( x0, x5,x10,x15) + QUARTERROUND( x1, x6,x11,x12) + QUARTERROUND( x2, x7, x8,x13) + QUARTERROUND( x3, x4, x9,x14) + } + x0 = PLUS(x0,j0); + x1 = PLUS(x1,j1); + x2 = PLUS(x2,j2); + x3 = PLUS(x3,j3); + x4 = PLUS(x4,j4); + x5 = PLUS(x5,j5); + x6 = PLUS(x6,j6); + x7 = PLUS(x7,j7); + x8 = PLUS(x8,j8); + x9 = PLUS(x9,j9); + x10 = PLUS(x10,j10); + x11 = PLUS(x11,j11); + x12 = PLUS(x12,j12); + x13 = PLUS(x13,j13); + x14 = PLUS(x14,j14); + x15 = PLUS(x15,j15); + + x0 = XOR(x0,U8TO32_LITTLE(m + 0)); + x1 = XOR(x1,U8TO32_LITTLE(m + 4)); + x2 = XOR(x2,U8TO32_LITTLE(m + 8)); + x3 = XOR(x3,U8TO32_LITTLE(m + 12)); + x4 = XOR(x4,U8TO32_LITTLE(m + 16)); + x5 = XOR(x5,U8TO32_LITTLE(m + 20)); + x6 = XOR(x6,U8TO32_LITTLE(m + 24)); + x7 = XOR(x7,U8TO32_LITTLE(m + 28)); + x8 = XOR(x8,U8TO32_LITTLE(m + 32)); + x9 = XOR(x9,U8TO32_LITTLE(m + 36)); + x10 = XOR(x10,U8TO32_LITTLE(m + 40)); + x11 = XOR(x11,U8TO32_LITTLE(m + 44)); + x12 = XOR(x12,U8TO32_LITTLE(m + 48)); + x13 = XOR(x13,U8TO32_LITTLE(m + 52)); + x14 = XOR(x14,U8TO32_LITTLE(m + 56)); + x15 = XOR(x15,U8TO32_LITTLE(m + 60)); + + j12 = PLUSONE(j12); + if (!j12) { + j13 = PLUSONE(j13); + /* stopping at 2^70 bytes per nonce is user's responsibility */ + } + + U32TO8_LITTLE(c + 0,x0); + U32TO8_LITTLE(c + 4,x1); + U32TO8_LITTLE(c + 8,x2); + U32TO8_LITTLE(c + 12,x3); + U32TO8_LITTLE(c + 16,x4); + U32TO8_LITTLE(c + 20,x5); + U32TO8_LITTLE(c + 24,x6); + U32TO8_LITTLE(c + 28,x7); + U32TO8_LITTLE(c + 32,x8); + U32TO8_LITTLE(c + 36,x9); + U32TO8_LITTLE(c + 40,x10); + U32TO8_LITTLE(c + 44,x11); + U32TO8_LITTLE(c + 48,x12); + U32TO8_LITTLE(c + 52,x13); + U32TO8_LITTLE(c + 56,x14); + U32TO8_LITTLE(c + 60,x15); + + if (bytes <= 64) { + if (bytes < 64) { + for (i = 0;i < bytes;++i) ctarget[i] = c[i]; + } + x->input[12] = j12; + x->input[13] = j13; + return; + } + bytes -= 64; + c += 64; + m += 64; + } +} diff --git a/freebsd/sys/crypto/chacha20/chacha.h b/freebsd/sys/crypto/chacha20/chacha.h new file mode 100644 index 00000000..02106eaa --- /dev/null +++ b/freebsd/sys/crypto/chacha20/chacha.h @@ -0,0 +1,32 @@ +/* $OpenBSD: chacha.h,v 1.4 2016/08/27 04:04:56 guenther Exp $ */ + +/* +chacha-merged.c version 20080118 +D. J. Bernstein +Public domain. + + $FreeBSD$ +*/ + +#ifndef CHACHA_H +#define CHACHA_H + +#include <sys/types.h> + +struct chacha_ctx { + u_int input[16]; +}; + +#define CHACHA_MINKEYLEN 16 +#define CHACHA_NONCELEN 8 +#define CHACHA_CTRLEN 8 +#define CHACHA_STATELEN (CHACHA_NONCELEN+CHACHA_CTRLEN) +#define CHACHA_BLOCKLEN 64 + +void chacha_keysetup(struct chacha_ctx *x, const u_char *k, u_int kbits); +void chacha_ivsetup(struct chacha_ctx *x, const u_char *iv, const u_char *ctr); +void chacha_encrypt_bytes(struct chacha_ctx *x, const u_char *m, + u_char *c, u_int bytes); + +#endif /* CHACHA_H */ + diff --git a/freebsd/sys/dev/e1000/e1000_82575.c b/freebsd/sys/dev/e1000/e1000_82575.c index 9958d98c..bffa1117 100644 --- a/freebsd/sys/dev/e1000/e1000_82575.c +++ b/freebsd/sys/dev/e1000/e1000_82575.c @@ -1681,7 +1681,7 @@ static s32 e1000_setup_serdes_link_82575(struct e1000_hw *hw) case E1000_CTRL_EXT_LINK_MODE_1000BASE_KX: /* disable PCS autoneg and support parallel detect only */ pcs_autoneg = FALSE; - /* fall through to default case */ + /* FALLTHROUGH */ default: if (hw->mac.type == e1000_82575 || hw->mac.type == e1000_82576) { @@ -1808,6 +1808,7 @@ static s32 e1000_get_media_type_82575(struct e1000_hw *hw) break; } /* fall through for I2C based SGMII */ + /* FALLTHROUGH */ case E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES: /* read media type from SFP EEPROM */ ret_val = e1000_set_sfp_media_type_82575(hw); diff --git a/freebsd/sys/dev/e1000/e1000_mbx.c b/freebsd/sys/dev/e1000/e1000_mbx.c index dc1482da..5503848f 100644 --- a/freebsd/sys/dev/e1000/e1000_mbx.c +++ b/freebsd/sys/dev/e1000/e1000_mbx.c @@ -781,6 +781,7 @@ s32 e1000_init_mbx_params_pf(struct e1000_hw *hw) mbx->stats.reqs = 0; mbx->stats.acks = 0; mbx->stats.rsts = 0; + /* FALLTHROUGH */ default: return E1000_SUCCESS; } diff --git a/freebsd/sys/dev/e1000/e1000_phy.c b/freebsd/sys/dev/e1000/e1000_phy.c index c656a6f1..59ec184e 100644 --- a/freebsd/sys/dev/e1000/e1000_phy.c +++ b/freebsd/sys/dev/e1000/e1000_phy.c @@ -1300,6 +1300,7 @@ s32 e1000_copper_link_setup_m88_gen2(struct e1000_hw *hw) phy_data |= M88E1000_PSCR_AUTO_X_1000T; break; } + /* FALLTHROUGH */ case 0: default: phy_data |= M88E1000_PSCR_AUTO_X_MODE; diff --git a/freebsd/sys/dev/e1000/e1000_vf.c b/freebsd/sys/dev/e1000/e1000_vf.c index 26f266e6..a3b4e69f 100644 --- a/freebsd/sys/dev/e1000/e1000_vf.c +++ b/freebsd/sys/dev/e1000/e1000_vf.c @@ -490,8 +490,10 @@ s32 e1000_promisc_set_vf(struct e1000_hw *hw, enum e1000_promisc_type type) break; case e1000_promisc_enabled: msgbuf |= E1000_VF_SET_PROMISC_MULTICAST; + /* FALLTHROUGH */ case e1000_promisc_unicast: msgbuf |= E1000_VF_SET_PROMISC_UNICAST; + /* FALLTHROUGH */ case e1000_promisc_disabled: break; default: diff --git a/freebsd/sys/dev/e1000/em_txrx.c b/freebsd/sys/dev/e1000/em_txrx.c index 9286f0ea..92b4a5f4 100644 --- a/freebsd/sys/dev/e1000/em_txrx.c +++ b/freebsd/sys/dev/e1000/em_txrx.c @@ -1,7 +1,8 @@ #include <machine/rtems-bsd-kernel-space.h> /*- - * Copyright (c) 2016-2017 Matthew Macy <mmacy@mattmacy.io> + * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> + * Copyright (c) 2017 Matthew Macy <mmacy@mattmacy.io> * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/freebsd/sys/dev/e1000/if_em.c b/freebsd/sys/dev/e1000/if_em.c index 94159e31..5150eaea 100644 --- a/freebsd/sys/dev/e1000/if_em.c +++ b/freebsd/sys/dev/e1000/if_em.c @@ -3,7 +3,7 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * - * Copyright (c) 2016 Matthew Macy <mmacy@mattmacy.io> + * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/freebsd/sys/dev/e1000/if_em.h b/freebsd/sys/dev/e1000/if_em.h index 0a6a09ca..0c892947 100644 --- a/freebsd/sys/dev/e1000/if_em.h +++ b/freebsd/sys/dev/e1000/if_em.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * - * Copyright (c) 2016 Matthew Macy <mmacy@mattmacy.io> + * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/freebsd/sys/dev/fdt/fdt_common.c b/freebsd/sys/dev/fdt/fdt_common.c index daa45832..de4a44b9 100644 --- a/freebsd/sys/dev/fdt/fdt_common.c +++ b/freebsd/sys/dev/fdt/fdt_common.c @@ -217,13 +217,13 @@ fdt_immr_addr(vm_offset_t immr_va) /* * Try to access the SOC node directly i.e. through /aliases/. */ - if ((node = OF_finddevice("soc")) != 0) + if ((node = OF_finddevice("soc")) != -1) if (fdt_is_compatible(node, "simple-bus")) goto moveon; /* * Find the node the long way. */ - if ((node = OF_finddevice("/")) == 0) + if ((node = OF_finddevice("/")) == -1) return (ENXIO); if ((node = fdt_find_compatible(node, "simple-bus", 0)) == 0) diff --git a/freebsd/sys/dev/gpio/gpiobusvar.h b/freebsd/sys/dev/gpio/gpiobusvar.h index fbcfbb4e..29677298 100644 --- a/freebsd/sys/dev/gpio/gpiobusvar.h +++ b/freebsd/sys/dev/gpio/gpiobusvar.h @@ -136,6 +136,8 @@ int gpio_pin_get_by_ofw_idx(device_t consumer, phandle_t node, int idx, gpio_pin_t *gpio); int gpio_pin_get_by_ofw_property(device_t consumer, phandle_t node, char *name, gpio_pin_t *gpio); +int gpio_pin_get_by_ofw_propidx(device_t consumer, phandle_t node, + char *name, int idx, gpio_pin_t *gpio); void gpio_pin_release(gpio_pin_t gpio); int gpio_pin_getcaps(gpio_pin_t pin, uint32_t *caps); int gpio_pin_is_active(gpio_pin_t pin, bool *active); diff --git a/freebsd/sys/dev/gpio/ofw_gpiobus.c b/freebsd/sys/dev/gpio/ofw_gpiobus.c index 70495a2a..1a87121c 100644 --- a/freebsd/sys/dev/gpio/ofw_gpiobus.c +++ b/freebsd/sys/dev/gpio/ofw_gpiobus.c @@ -65,8 +65,8 @@ static int ofw_gpiobus_parse_gpios_impl(device_t, phandle_t, char *, * tree consumers. * */ -static int -gpio_pin_get_by_ofw_impl(device_t consumer, phandle_t cnode, +int +gpio_pin_get_by_ofw_propidx(device_t consumer, phandle_t cnode, char *prop_name, int idx, gpio_pin_t *out_pin) { phandle_t xref; @@ -120,7 +120,7 @@ gpio_pin_get_by_ofw_idx(device_t consumer, phandle_t node, int idx, gpio_pin_t *pin) { - return (gpio_pin_get_by_ofw_impl(consumer, node, "gpios", idx, pin)); + return (gpio_pin_get_by_ofw_propidx(consumer, node, "gpios", idx, pin)); } int @@ -128,7 +128,7 @@ gpio_pin_get_by_ofw_property(device_t consumer, phandle_t node, char *name, gpio_pin_t *pin) { - return (gpio_pin_get_by_ofw_impl(consumer, node, name, 0, pin)); + return (gpio_pin_get_by_ofw_propidx(consumer, node, name, 0, pin)); } int diff --git a/freebsd/sys/dev/nvme/nvme.h b/freebsd/sys/dev/nvme/nvme.h index 5d11f793..b51b4a97 100644 --- a/freebsd/sys/dev/nvme/nvme.h +++ b/freebsd/sys/dev/nvme/nvme.h @@ -36,6 +36,7 @@ #endif #include <sys/param.h> +#include <sys/endian.h> #define NVME_PASSTHROUGH_CMD _IOWR('n', 0, struct nvme_pt_command) #define NVME_RESET_CONTROLLER _IO('n', 1) @@ -59,153 +60,358 @@ /* Cap nvme to 1MB transfers driver explodes with larger sizes */ #define NVME_MAX_XFER_SIZE (MAXPHYS < (1<<20) ? MAXPHYS : (1<<20)) -union cap_lo_register { - uint32_t raw; - struct { - /** maximum queue entries supported */ - uint32_t mqes : 16; - - /** contiguous queues required */ - uint32_t cqr : 1; - - /** arbitration mechanism supported */ - uint32_t ams : 2; - - uint32_t reserved1 : 5; - - /** timeout */ - uint32_t to : 8; - } bits __packed; -} __packed; - -_Static_assert(sizeof(union cap_lo_register) == 4, "bad size for cap_lo_register"); - -union cap_hi_register { - uint32_t raw; - struct { - /** doorbell stride */ - uint32_t dstrd : 4; - - uint32_t reserved3 : 1; - - /** command sets supported */ - uint32_t css_nvm : 1; - - uint32_t css_reserved : 3; - uint32_t reserved2 : 7; - - /** memory page size minimum */ - uint32_t mpsmin : 4; - - /** memory page size maximum */ - uint32_t mpsmax : 4; - - uint32_t reserved1 : 8; - } bits __packed; -} __packed; - -_Static_assert(sizeof(union cap_hi_register) == 4, "bad size of cap_hi_register"); - -union cc_register { - uint32_t raw; - struct { - /** enable */ - uint32_t en : 1; - - uint32_t reserved1 : 3; - - /** i/o command set selected */ - uint32_t css : 3; - - /** memory page size */ - uint32_t mps : 4; - - /** arbitration mechanism selected */ - uint32_t ams : 3; - - /** shutdown notification */ - uint32_t shn : 2; - - /** i/o submission queue entry size */ - uint32_t iosqes : 4; - - /** i/o completion queue entry size */ - uint32_t iocqes : 4; - - uint32_t reserved2 : 8; - } bits __packed; -} __packed; +/* Register field definitions */ +#define NVME_CAP_LO_REG_MQES_SHIFT (0) +#define NVME_CAP_LO_REG_MQES_MASK (0xFFFF) +#define NVME_CAP_LO_REG_CQR_SHIFT (16) +#define NVME_CAP_LO_REG_CQR_MASK (0x1) +#define NVME_CAP_LO_REG_AMS_SHIFT (17) +#define NVME_CAP_LO_REG_AMS_MASK (0x3) +#define NVME_CAP_LO_REG_TO_SHIFT (24) +#define NVME_CAP_LO_REG_TO_MASK (0xFF) + +#define NVME_CAP_HI_REG_DSTRD_SHIFT (0) +#define NVME_CAP_HI_REG_DSTRD_MASK (0xF) +#define NVME_CAP_HI_REG_CSS_NVM_SHIFT (5) +#define NVME_CAP_HI_REG_CSS_NVM_MASK (0x1) +#define NVME_CAP_HI_REG_MPSMIN_SHIFT (16) +#define NVME_CAP_HI_REG_MPSMIN_MASK (0xF) +#define NVME_CAP_HI_REG_MPSMAX_SHIFT (20) +#define NVME_CAP_HI_REG_MPSMAX_MASK (0xF) + +#define NVME_CC_REG_EN_SHIFT (0) +#define NVME_CC_REG_EN_MASK (0x1) +#define NVME_CC_REG_CSS_SHIFT (4) +#define NVME_CC_REG_CSS_MASK (0x7) +#define NVME_CC_REG_MPS_SHIFT (7) +#define NVME_CC_REG_MPS_MASK (0xF) +#define NVME_CC_REG_AMS_SHIFT (11) +#define NVME_CC_REG_AMS_MASK (0x7) +#define NVME_CC_REG_SHN_SHIFT (14) +#define NVME_CC_REG_SHN_MASK (0x3) +#define NVME_CC_REG_IOSQES_SHIFT (16) +#define NVME_CC_REG_IOSQES_MASK (0xF) +#define NVME_CC_REG_IOCQES_SHIFT (20) +#define NVME_CC_REG_IOCQES_MASK (0xF) + +#define NVME_CSTS_REG_RDY_SHIFT (0) +#define NVME_CSTS_REG_RDY_MASK (0x1) +#define NVME_CSTS_REG_CFS_SHIFT (1) +#define NVME_CSTS_REG_CFS_MASK (0x1) +#define NVME_CSTS_REG_SHST_SHIFT (2) +#define NVME_CSTS_REG_SHST_MASK (0x3) + +#define NVME_CSTS_GET_SHST(csts) (((csts) >> NVME_CSTS_REG_SHST_SHIFT) & NVME_CSTS_REG_SHST_MASK) + +#define NVME_AQA_REG_ASQS_SHIFT (0) +#define NVME_AQA_REG_ASQS_MASK (0xFFF) +#define NVME_AQA_REG_ACQS_SHIFT (16) +#define NVME_AQA_REG_ACQS_MASK (0xFFF) + +/* Command field definitions */ + +#define NVME_CMD_OPC_SHIFT (0) +#define NVME_CMD_OPC_MASK (0xFF) +#define NVME_CMD_FUSE_SHIFT (8) +#define NVME_CMD_FUSE_MASK (0x3) + +#define NVME_CMD_SET_OPC(opc) (htole16(((opc) & NVME_CMD_OPC_MASK) << NVME_CMD_OPC_SHIFT)) + +#define NVME_STATUS_P_SHIFT (0) +#define NVME_STATUS_P_MASK (0x1) +#define NVME_STATUS_SC_SHIFT (1) +#define NVME_STATUS_SC_MASK (0xFF) +#define NVME_STATUS_SCT_SHIFT (9) +#define NVME_STATUS_SCT_MASK (0x7) +#define NVME_STATUS_M_SHIFT (14) +#define NVME_STATUS_M_MASK (0x1) +#define NVME_STATUS_DNR_SHIFT (15) +#define NVME_STATUS_DNR_MASK (0x1) + +#define NVME_STATUS_GET_P(st) (((st) >> NVME_STATUS_P_SHIFT) & NVME_STATUS_P_MASK) +#define NVME_STATUS_GET_SC(st) (((st) >> NVME_STATUS_SC_SHIFT) & NVME_STATUS_SC_MASK) +#define NVME_STATUS_GET_SCT(st) (((st) >> NVME_STATUS_SCT_SHIFT) & NVME_STATUS_SCT_MASK) +#define NVME_STATUS_GET_M(st) (((st) >> NVME_STATUS_M_SHIFT) & NVME_STATUS_M_MASK) +#define NVME_STATUS_GET_DNR(st) (((st) >> NVME_STATUS_DNR_SHIFT) & NVME_STATUS_DNR_MASK) + +#define NVME_PWR_ST_MPS_SHIFT (0) +#define NVME_PWR_ST_MPS_MASK (0x1) +#define NVME_PWR_ST_NOPS_SHIFT (1) +#define NVME_PWR_ST_NOPS_MASK (0x1) +#define NVME_PWR_ST_RRT_SHIFT (0) +#define NVME_PWR_ST_RRT_MASK (0x1F) +#define NVME_PWR_ST_RRL_SHIFT (0) +#define NVME_PWR_ST_RRL_MASK (0x1F) +#define NVME_PWR_ST_RWT_SHIFT (0) +#define NVME_PWR_ST_RWT_MASK (0x1F) +#define NVME_PWR_ST_RWL_SHIFT (0) +#define NVME_PWR_ST_RWL_MASK (0x1F) +#define NVME_PWR_ST_IPS_SHIFT (6) +#define NVME_PWR_ST_IPS_MASK (0x3) +#define NVME_PWR_ST_APW_SHIFT (0) +#define NVME_PWR_ST_APW_MASK (0x7) +#define NVME_PWR_ST_APS_SHIFT (6) +#define NVME_PWR_ST_APS_MASK (0x3) + +/** Controller Multi-path I/O and Namespace Sharing Capabilities */ +/* More then one port */ +#define NVME_CTRLR_DATA_MIC_MPORTS_SHIFT (0) +#define NVME_CTRLR_DATA_MIC_MPORTS_MASK (0x1) +/* More then one controller */ +#define NVME_CTRLR_DATA_MIC_MCTRLRS_SHIFT (1) +#define NVME_CTRLR_DATA_MIC_MCTRLRS_MASK (0x1) +/* SR-IOV Virtual Function */ +#define NVME_CTRLR_DATA_MIC_SRIOVVF_SHIFT (2) +#define NVME_CTRLR_DATA_MIC_SRIOVVF_MASK (0x1) + +/** OACS - optional admin command support */ +/* supports security send/receive commands */ +#define NVME_CTRLR_DATA_OACS_SECURITY_SHIFT (0) +#define NVME_CTRLR_DATA_OACS_SECURITY_MASK (0x1) +/* supports format nvm command */ +#define NVME_CTRLR_DATA_OACS_FORMAT_SHIFT (1) +#define NVME_CTRLR_DATA_OACS_FORMAT_MASK (0x1) +/* supports firmware activate/download commands */ +#define NVME_CTRLR_DATA_OACS_FIRMWARE_SHIFT (2) +#define NVME_CTRLR_DATA_OACS_FIRMWARE_MASK (0x1) +/* supports namespace management commands */ +#define NVME_CTRLR_DATA_OACS_NSMGMT_SHIFT (3) +#define NVME_CTRLR_DATA_OACS_NSMGMT_MASK (0x1) +/* supports Device Self-test command */ +#define NVME_CTRLR_DATA_OACS_SELFTEST_SHIFT (4) +#define NVME_CTRLR_DATA_OACS_SELFTEST_MASK (0x1) +/* supports Directives */ +#define NVME_CTRLR_DATA_OACS_DIRECTIVES_SHIFT (5) +#define NVME_CTRLR_DATA_OACS_DIRECTIVES_MASK (0x1) +/* supports NVMe-MI Send/Receive */ +#define NVME_CTRLR_DATA_OACS_NVMEMI_SHIFT (6) +#define NVME_CTRLR_DATA_OACS_NVMEMI_MASK (0x1) +/* supports Virtualization Management */ +#define NVME_CTRLR_DATA_OACS_VM_SHIFT (7) +#define NVME_CTRLR_DATA_OACS_VM_MASK (0x1) +/* supports Doorbell Buffer Config */ +#define NVME_CTRLR_DATA_OACS_DBBUFFER_SHIFT (8) +#define NVME_CTRLR_DATA_OACS_DBBUFFER_MASK (0x1) + +/** firmware updates */ +/* first slot is read-only */ +#define NVME_CTRLR_DATA_FRMW_SLOT1_RO_SHIFT (0) +#define NVME_CTRLR_DATA_FRMW_SLOT1_RO_MASK (0x1) +/* number of firmware slots */ +#define NVME_CTRLR_DATA_FRMW_NUM_SLOTS_SHIFT (1) +#define NVME_CTRLR_DATA_FRMW_NUM_SLOTS_MASK (0x7) + +/** log page attributes */ +/* per namespace smart/health log page */ +#define NVME_CTRLR_DATA_LPA_NS_SMART_SHIFT (0) +#define NVME_CTRLR_DATA_LPA_NS_SMART_MASK (0x1) + +/** AVSCC - admin vendor specific command configuration */ +/* admin vendor specific commands use spec format */ +#define NVME_CTRLR_DATA_AVSCC_SPEC_FORMAT_SHIFT (0) +#define NVME_CTRLR_DATA_AVSCC_SPEC_FORMAT_MASK (0x1) + +/** Autonomous Power State Transition Attributes */ +/* Autonomous Power State Transitions supported */ +#define NVME_CTRLR_DATA_APSTA_APST_SUPP_SHIFT (0) +#define NVME_CTRLR_DATA_APSTA_APST_SUPP_MASK (0x1) + +/** submission queue entry size */ +#define NVME_CTRLR_DATA_SQES_MIN_SHIFT (0) +#define NVME_CTRLR_DATA_SQES_MIN_MASK (0xF) +#define NVME_CTRLR_DATA_SQES_MAX_SHIFT (4) +#define NVME_CTRLR_DATA_SQES_MAX_MASK (0xF) + +/** completion queue entry size */ +#define NVME_CTRLR_DATA_CQES_MIN_SHIFT (0) +#define NVME_CTRLR_DATA_CQES_MIN_MASK (0xF) +#define NVME_CTRLR_DATA_CQES_MAX_SHIFT (4) +#define NVME_CTRLR_DATA_CQES_MAX_MASK (0xF) + +/** optional nvm command support */ +#define NVME_CTRLR_DATA_ONCS_COMPARE_SHIFT (0) +#define NVME_CTRLR_DATA_ONCS_COMPARE_MASK (0x1) +#define NVME_CTRLR_DATA_ONCS_WRITE_UNC_SHIFT (1) +#define NVME_CTRLR_DATA_ONCS_WRITE_UNC_MASK (0x1) +#define NVME_CTRLR_DATA_ONCS_DSM_SHIFT (2) +#define NVME_CTRLR_DATA_ONCS_DSM_MASK (0x1) +#define NVME_CTRLR_DATA_ONCS_WRZERO_SHIFT (3) +#define NVME_CTRLR_DATA_ONCS_WRZERO_MASK (0x1) +#define NVME_CTRLR_DATA_ONCS_SAVEFEAT_SHIFT (4) +#define NVME_CTRLR_DATA_ONCS_SAVEFEAT_MASK (0x1) +#define NVME_CTRLR_DATA_ONCS_RESERV_SHIFT (5) +#define NVME_CTRLR_DATA_ONCS_RESERV_MASK (0x1) +#define NVME_CTRLR_DATA_ONCS_TIMESTAMP_SHIFT (6) +#define NVME_CTRLR_DATA_ONCS_TIMESTAMP_MASK (0x1) + +/** Fused Operation Support */ +#define NVME_CTRLR_DATA_FUSES_CNW_SHIFT (0) +#define NVME_CTRLR_DATA_FUSES_CNW_MASK (0x1) + +/** Format NVM Attributes */ +#define NVME_CTRLR_DATA_FNA_FORMAT_ALL_SHIFT (0) +#define NVME_CTRLR_DATA_FNA_FORMAT_ALL_MASK (0x1) +#define NVME_CTRLR_DATA_FNA_ERASE_ALL_SHIFT (1) +#define NVME_CTRLR_DATA_FNA_ERASE_ALL_MASK (0x1) +#define NVME_CTRLR_DATA_FNA_CRYPTO_ERASE_SHIFT (2) +#define NVME_CTRLR_DATA_FNA_CRYPTO_ERASE_MASK (0x1) + +/** volatile write cache */ +#define NVME_CTRLR_DATA_VWC_PRESENT_SHIFT (0) +#define NVME_CTRLR_DATA_VWC_PRESENT_MASK (0x1) + +/** namespace features */ +/* thin provisioning */ +#define NVME_NS_DATA_NSFEAT_THIN_PROV_SHIFT (0) +#define NVME_NS_DATA_NSFEAT_THIN_PROV_MASK (0x1) +/* NAWUN, NAWUPF, and NACWU fields are valid */ +#define NVME_NS_DATA_NSFEAT_NA_FIELDS_SHIFT (1) +#define NVME_NS_DATA_NSFEAT_NA_FIELDS_MASK (0x1) +/* Deallocated or Unwritten Logical Block errors supported */ +#define NVME_NS_DATA_NSFEAT_DEALLOC_SHIFT (2) +#define NVME_NS_DATA_NSFEAT_DEALLOC_MASK (0x1) +/* NGUID and EUI64 fields are not reusable */ +#define NVME_NS_DATA_NSFEAT_NO_ID_REUSE_SHIFT (3) +#define NVME_NS_DATA_NSFEAT_NO_ID_REUSE_MASK (0x1) + +/** formatted lba size */ +#define NVME_NS_DATA_FLBAS_FORMAT_SHIFT (0) +#define NVME_NS_DATA_FLBAS_FORMAT_MASK (0xF) +#define NVME_NS_DATA_FLBAS_EXTENDED_SHIFT (4) +#define NVME_NS_DATA_FLBAS_EXTENDED_MASK (0x1) + +/** metadata capabilities */ +/* metadata can be transferred as part of data prp list */ +#define NVME_NS_DATA_MC_EXTENDED_SHIFT (0) +#define NVME_NS_DATA_MC_EXTENDED_MASK (0x1) +/* metadata can be transferred with separate metadata pointer */ +#define NVME_NS_DATA_MC_POINTER_SHIFT (1) +#define NVME_NS_DATA_MC_POINTER_MASK (0x1) + +/** end-to-end data protection capabilities */ +/* protection information type 1 */ +#define NVME_NS_DATA_DPC_PIT1_SHIFT (0) +#define NVME_NS_DATA_DPC_PIT1_MASK (0x1) +/* protection information type 2 */ +#define NVME_NS_DATA_DPC_PIT2_SHIFT (1) +#define NVME_NS_DATA_DPC_PIT2_MASK (0x1) +/* protection information type 3 */ +#define NVME_NS_DATA_DPC_PIT3_SHIFT (2) +#define NVME_NS_DATA_DPC_PIT3_MASK (0x1) +/* first eight bytes of metadata */ +#define NVME_NS_DATA_DPC_MD_START_SHIFT (3) +#define NVME_NS_DATA_DPC_MD_START_MASK (0x1) +/* last eight bytes of metadata */ +#define NVME_NS_DATA_DPC_MD_END_SHIFT (4) +#define NVME_NS_DATA_DPC_MD_END_MASK (0x1) + +/** end-to-end data protection type settings */ +/* protection information type */ +#define NVME_NS_DATA_DPS_PIT_SHIFT (0) +#define NVME_NS_DATA_DPS_PIT_MASK (0x7) +/* 1 == protection info transferred at start of metadata */ +/* 0 == protection info transferred at end of metadata */ +#define NVME_NS_DATA_DPS_MD_START_SHIFT (3) +#define NVME_NS_DATA_DPS_MD_START_MASK (0x1) + +/** Namespace Multi-path I/O and Namespace Sharing Capabilities */ +/* the namespace may be attached to two or more controllers */ +#define NVME_NS_DATA_NMIC_MAY_BE_SHARED_SHIFT (0) +#define NVME_NS_DATA_NMIC_MAY_BE_SHARED_MASK (0x1) + +/** Reservation Capabilities */ +/* Persist Through Power Loss */ +#define NVME_NS_DATA_RESCAP_PTPL_SHIFT (0) +#define NVME_NS_DATA_RESCAP_PTPL_MASK (0x1) +/* supports the Write Exclusive */ +#define NVME_NS_DATA_RESCAP_WR_EX_SHIFT (1) +#define NVME_NS_DATA_RESCAP_WR_EX_MASK (0x1) +/* supports the Exclusive Access */ +#define NVME_NS_DATA_RESCAP_EX_AC_SHIFT (2) +#define NVME_NS_DATA_RESCAP_EX_AC_MASK (0x1) +/* supports the Write Exclusive – Registrants Only */ +#define NVME_NS_DATA_RESCAP_WR_EX_RO_SHIFT (3) +#define NVME_NS_DATA_RESCAP_WR_EX_RO_MASK (0x1) +/* supports the Exclusive Access - Registrants Only */ +#define NVME_NS_DATA_RESCAP_EX_AC_RO_SHIFT (4) +#define NVME_NS_DATA_RESCAP_EX_AC_RO_MASK (0x1) +/* supports the Write Exclusive – All Registrants */ +#define NVME_NS_DATA_RESCAP_WR_EX_AR_SHIFT (5) +#define NVME_NS_DATA_RESCAP_WR_EX_AR_MASK (0x1) +/* supports the Exclusive Access - All Registrants */ +#define NVME_NS_DATA_RESCAP_EX_AC_AR_SHIFT (6) +#define NVME_NS_DATA_RESCAP_EX_AC_AR_MASK (0x1) +/* Ignore Existing Key is used as defined in revision 1.3 or later */ +#define NVME_NS_DATA_RESCAP_IEKEY13_SHIFT (7) +#define NVME_NS_DATA_RESCAP_IEKEY13_MASK (0x1) + +/** Format Progress Indicator */ +/* percentage of the Format NVM command that remains to be completed */ +#define NVME_NS_DATA_FPI_PERC_SHIFT (0) +#define NVME_NS_DATA_FPI_PERC_MASK (0x7f) +/* namespace supports the Format Progress Indicator */ +#define NVME_NS_DATA_FPI_SUPP_SHIFT (7) +#define NVME_NS_DATA_FPI_SUPP_MASK (0x1) + +/** lba format support */ +/* metadata size */ +#define NVME_NS_DATA_LBAF_MS_SHIFT (0) +#define NVME_NS_DATA_LBAF_MS_MASK (0xFFFF) +/* lba data size */ +#define NVME_NS_DATA_LBAF_LBADS_SHIFT (16) +#define NVME_NS_DATA_LBAF_LBADS_MASK (0xFF) +/* relative performance */ +#define NVME_NS_DATA_LBAF_RP_SHIFT (24) +#define NVME_NS_DATA_LBAF_RP_MASK (0x3) + +enum nvme_critical_warning_state { + NVME_CRIT_WARN_ST_AVAILABLE_SPARE = 0x1, + NVME_CRIT_WARN_ST_TEMPERATURE = 0x2, + NVME_CRIT_WARN_ST_DEVICE_RELIABILITY = 0x4, + NVME_CRIT_WARN_ST_READ_ONLY = 0x8, + NVME_CRIT_WARN_ST_VOLATILE_MEMORY_BACKUP = 0x10, +}; +#define NVME_CRIT_WARN_ST_RESERVED_MASK (0xE0) -_Static_assert(sizeof(union cc_register) == 4, "bad size for cc_register"); +/* slot for current FW */ +#define NVME_FIRMWARE_PAGE_AFI_SLOT_SHIFT (0) +#define NVME_FIRMWARE_PAGE_AFI_SLOT_MASK (0x7) +/* CC register SHN field values */ enum shn_value { NVME_SHN_NORMAL = 0x1, NVME_SHN_ABRUPT = 0x2, }; -union csts_register { - uint32_t raw; - struct { - /** ready */ - uint32_t rdy : 1; - - /** controller fatal status */ - uint32_t cfs : 1; - - /** shutdown status */ - uint32_t shst : 2; - - uint32_t reserved1 : 28; - } bits __packed; -} __packed; - -_Static_assert(sizeof(union csts_register) == 4, "bad size for csts_register"); - +/* CSTS register SHST field values */ enum shst_value { NVME_SHST_NORMAL = 0x0, NVME_SHST_OCCURRING = 0x1, NVME_SHST_COMPLETE = 0x2, }; -union aqa_register { - uint32_t raw; - struct { - /** admin submission queue size */ - uint32_t asqs : 12; - - uint32_t reserved1 : 4; - - /** admin completion queue size */ - uint32_t acqs : 12; - - uint32_t reserved2 : 4; - } bits __packed; -} __packed; - -_Static_assert(sizeof(union aqa_register) == 4, "bad size for aqa_resgister"); - struct nvme_registers { /** controller capabilities */ - union cap_lo_register cap_lo; - union cap_hi_register cap_hi; + uint32_t cap_lo; + uint32_t cap_hi; uint32_t vs; /* version */ uint32_t intms; /* interrupt mask set */ uint32_t intmc; /* interrupt mask clear */ /** controller configuration */ - union cc_register cc; + uint32_t cc; uint32_t reserved1; /** controller status */ - union csts_register csts; + uint32_t csts; uint32_t reserved2; /** admin queue attributes */ - union aqa_register aqa; + uint32_t aqa; uint64_t asq; /* admin submission queue base addr */ uint64_t acq; /* admin completion queue base addr */ @@ -222,9 +428,7 @@ _Static_assert(sizeof(struct nvme_registers) == 0x1008, "bad size for nvme_regis struct nvme_command { /* dword 0 */ - uint16_t opc : 8; /* opcode */ - uint16_t fuse : 2; /* fused operation */ - uint16_t rsvd1 : 6; + uint16_t opc_fuse; /* opcode, fused operation */ uint16_t cid; /* command identifier */ /* dword 1 */ @@ -254,18 +458,6 @@ struct nvme_command _Static_assert(sizeof(struct nvme_command) == 16 * 4, "bad size for nvme_command"); -struct nvme_status { - - uint16_t p : 1; /* phase tag */ - uint16_t sc : 8; /* status code */ - uint16_t sct : 3; /* status code type */ - uint16_t rsvd2 : 2; - uint16_t m : 1; /* more */ - uint16_t dnr : 1; /* do not retry */ -} __packed; - -_Static_assert(sizeof(struct nvme_status) == 2, "bad size for nvme_status"); - struct nvme_completion { /* dword 0 */ @@ -280,18 +472,20 @@ struct nvme_completion { /* dword 3 */ uint16_t cid; /* command identifier */ - struct nvme_status status; + uint16_t status; } __packed; _Static_assert(sizeof(struct nvme_completion) == 4 * 4, "bad size for nvme_completion"); struct nvme_dsm_range { - uint32_t attributes; uint32_t length; uint64_t starting_lba; } __packed; +/* Largest DSM Trim that can be done */ +#define NVME_MAX_DSM_TRIM 4096 + _Static_assert(sizeof(struct nvme_dsm_range) == 16, "bad size for nvme_dsm_ranage"); /* status code types */ @@ -318,10 +512,31 @@ enum nvme_generic_command_status_code { NVME_SC_ABORTED_MISSING_FUSED = 0x0a, NVME_SC_INVALID_NAMESPACE_OR_FORMAT = 0x0b, NVME_SC_COMMAND_SEQUENCE_ERROR = 0x0c, + NVME_SC_INVALID_SGL_SEGMENT_DESCR = 0x0d, + NVME_SC_INVALID_NUMBER_OF_SGL_DESCR = 0x0e, + NVME_SC_DATA_SGL_LENGTH_INVALID = 0x0f, + NVME_SC_METADATA_SGL_LENGTH_INVALID = 0x10, + NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID = 0x11, + NVME_SC_INVALID_USE_OF_CMB = 0x12, + NVME_SC_PRP_OFFET_INVALID = 0x13, + NVME_SC_ATOMIC_WRITE_UNIT_EXCEEDED = 0x14, + NVME_SC_OPERATION_DENIED = 0x15, + NVME_SC_SGL_OFFSET_INVALID = 0x16, + /* 0x17 - reserved */ + NVME_SC_HOST_ID_INCONSISTENT_FORMAT = 0x18, + NVME_SC_KEEP_ALIVE_TIMEOUT_EXPIRED = 0x19, + NVME_SC_KEEP_ALIVE_TIMEOUT_INVALID = 0x1a, + NVME_SC_ABORTED_DUE_TO_PREEMPT = 0x1b, + NVME_SC_SANITIZE_FAILED = 0x1c, + NVME_SC_SANITIZE_IN_PROGRESS = 0x1d, + NVME_SC_SGL_DATA_BLOCK_GRAN_INVALID = 0x1e, + NVME_SC_NOT_SUPPORTED_IN_CMB = 0x1f, NVME_SC_LBA_OUT_OF_RANGE = 0x80, NVME_SC_CAPACITY_EXCEEDED = 0x81, NVME_SC_NAMESPACE_NOT_READY = 0x82, + NVME_SC_RESERVATION_CONFLICT = 0x83, + NVME_SC_FORMAT_IN_PROGRESS = 0x84, }; /* command specific status codes */ @@ -338,6 +553,29 @@ enum nvme_command_specific_status_code { NVME_SC_INVALID_LOG_PAGE = 0x09, NVME_SC_INVALID_FORMAT = 0x0a, NVME_SC_FIRMWARE_REQUIRES_RESET = 0x0b, + NVME_SC_INVALID_QUEUE_DELETION = 0x0c, + NVME_SC_FEATURE_NOT_SAVEABLE = 0x0d, + NVME_SC_FEATURE_NOT_CHANGEABLE = 0x0e, + NVME_SC_FEATURE_NOT_NS_SPECIFIC = 0x0f, + NVME_SC_FW_ACT_REQUIRES_NVMS_RESET = 0x10, + NVME_SC_FW_ACT_REQUIRES_RESET = 0x11, + NVME_SC_FW_ACT_REQUIRES_TIME = 0x12, + NVME_SC_FW_ACT_PROHIBITED = 0x13, + NVME_SC_OVERLAPPING_RANGE = 0x14, + NVME_SC_NS_INSUFFICIENT_CAPACITY = 0x15, + NVME_SC_NS_ID_UNAVAILABLE = 0x16, + /* 0x17 - reserved */ + NVME_SC_NS_ALREADY_ATTACHED = 0x18, + NVME_SC_NS_IS_PRIVATE = 0x19, + NVME_SC_NS_NOT_ATTACHED = 0x1a, + NVME_SC_THIN_PROV_NOT_SUPPORTED = 0x1b, + NVME_SC_CTRLR_LIST_INVALID = 0x1c, + NVME_SC_SELT_TEST_IN_PROGRESS = 0x1d, + NVME_SC_BOOT_PART_WRITE_PROHIB = 0x1e, + NVME_SC_INVALID_CTRLR_ID = 0x1f, + NVME_SC_INVALID_SEC_CTRLR_STATE = 0x20, + NVME_SC_INVALID_NUM_OF_CTRLR_RESRC = 0x21, + NVME_SC_INVALID_RESOURCE_ID = 0x22, NVME_SC_CONFLICTING_ATTRIBUTES = 0x80, NVME_SC_INVALID_PROTECTION_INFO = 0x81, @@ -353,6 +591,7 @@ enum nvme_media_error_status_code { NVME_SC_REFERENCE_TAG_CHECK_ERROR = 0x84, NVME_SC_COMPARE_FAILURE = 0x85, NVME_SC_ACCESS_DENIED = 0x86, + NVME_SC_DEALLOCATED_OR_UNWRITTEN = 0x87, }; /* admin opcodes */ @@ -374,11 +613,20 @@ enum nvme_admin_opcode { /* 0x0e-0x0f - reserved */ NVME_OPC_FIRMWARE_ACTIVATE = 0x10, NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD = 0x11, + NVME_OPC_DEVICE_SELF_TEST = 0x14, NVME_OPC_NAMESPACE_ATTACHMENT = 0x15, + NVME_OPC_KEEP_ALIVE = 0x18, + NVME_OPC_DIRECTIVE_SEND = 0x19, + NVME_OPC_DIRECTIVE_RECEIVE = 0x1a, + NVME_OPC_VIRTUALIZATION_MANAGEMENT = 0x1c, + NVME_OPC_NVME_MI_SEND = 0x1d, + NVME_OPC_NVME_MI_RECEIVE = 0x1e, + NVME_OPC_DOORBELL_BUFFER_CONFIG = 0x7c, NVME_OPC_FORMAT_NVM = 0x80, NVME_OPC_SECURITY_SEND = 0x81, NVME_OPC_SECURITY_RECEIVE = 0x82, + NVME_OPC_SANITIZE = 0x84, }; /* nvme nvm opcodes */ @@ -389,8 +637,17 @@ enum nvme_nvm_opcode { /* 0x03 - reserved */ NVME_OPC_WRITE_UNCORRECTABLE = 0x04, NVME_OPC_COMPARE = 0x05, - /* 0x06-0x07 - reserved */ + /* 0x06 - reserved */ + NVME_OPC_WRITE_ZEROES = 0x08, + /* 0x07 - reserved */ NVME_OPC_DATASET_MANAGEMENT = 0x09, + /* 0x0a-0x0c - reserved */ + NVME_OPC_RESERVATION_REGISTER = 0x0d, + NVME_OPC_RESERVATION_REPORT = 0x0e, + /* 0x0f-0x10 - reserved */ + NVME_OPC_RESERVATION_ACQUIRE = 0x11, + /* 0x12-0x14 - reserved */ + NVME_OPC_RESERVATION_RELEASE = 0x15, }; enum nvme_feature { @@ -435,27 +692,22 @@ struct nvme_power_state { /** Maximum Power */ uint16_t mp; /* Maximum Power */ uint8_t ps_rsvd1; - uint8_t mps : 1; /* Max Power Scale */ - uint8_t nops : 1; /* Non-Operational State */ - uint8_t ps_rsvd2 : 6; + uint8_t mps_nops; /* Max Power Scale, Non-Operational State */ + uint32_t enlat; /* Entry Latency */ uint32_t exlat; /* Exit Latency */ - uint8_t rrt : 5; /* Relative Read Throughput */ - uint8_t ps_rsvd3 : 3; - uint8_t rrl : 5; /* Relative Read Latency */ - uint8_t ps_rsvd4 : 3; - uint8_t rwt : 5; /* Relative Write Throughput */ - uint8_t ps_rsvd5 : 3; - uint8_t rwl : 5; /* Relative Write Latency */ - uint8_t ps_rsvd6 : 3; + + uint8_t rrt; /* Relative Read Throughput */ + uint8_t rrl; /* Relative Read Latency */ + uint8_t rwt; /* Relative Write Throughput */ + uint8_t rwl; /* Relative Write Latency */ + uint16_t idlp; /* Idle Power */ - uint8_t ps_rsvd7 : 6; - uint8_t ips : 2; /* Idle Power Scale */ + uint8_t ips; /* Idle Power Scale */ uint8_t ps_rsvd8; + uint16_t actp; /* Active Power */ - uint8_t apw : 3; /* Active Power Workload */ - uint8_t ps_rsvd9 : 3; - uint8_t aps : 2; /* Active Power Scale */ + uint8_t apw_aps; /* Active Power Workload, Active Power Scale */ uint8_t ps_rsvd10[9]; } __packed; @@ -524,21 +776,7 @@ struct nvme_controller_data { /* bytes 256-511: admin command set attributes */ /** optional admin command support */ - struct { - /* supports security send/receive commands */ - uint16_t security : 1; - - /* supports format nvm command */ - uint16_t format : 1; - - /* supports firmware activate/download commands */ - uint16_t firmware : 1; - - /* supports namespace management commands */ - uint16_t nsmgmt : 1; - - uint16_t oacs_rsvd : 12; - } __packed oacs; + uint16_t oacs; /** abort command limit */ uint8_t acl; @@ -547,23 +785,10 @@ struct nvme_controller_data { uint8_t aerl; /** firmware updates */ - struct { - /* first slot is read-only */ - uint8_t slot1_ro : 1; - - /* number of firmware slots */ - uint8_t num_slots : 3; - - uint8_t frmw_rsvd : 4; - } __packed frmw; + uint8_t frmw; /** log page attributes */ - struct { - /* per namespace smart/health log page */ - uint8_t ns_smart : 1; - - uint8_t lpa_rsvd : 7; - } __packed lpa; + uint8_t lpa; /** error log page entries */ uint8_t elpe; @@ -572,20 +797,10 @@ struct nvme_controller_data { uint8_t npss; /** admin vendor specific command configuration */ - struct { - /* admin vendor specific commands use spec format */ - uint8_t spec_format : 1; - - uint8_t avscc_rsvd : 7; - } __packed avscc; + uint8_t avscc; /** Autonomous Power State Transition Attributes */ - struct { - /* Autonmous Power State Transitions supported */ - uint8_t apst_supp : 1; - - uint8_t apsta_rsvd : 7; - } __packed apsta; + uint8_t apsta; /** Warning Composite Temperature Threshold */ uint16_t wctemp; @@ -636,20 +851,14 @@ struct nvme_controller_data { /** Sanitize Capabilities */ uint32_t sanicap; /* Really a bitfield */ - uint8_t reserved3[180]; + uint8_t reserved3[180]; /* bytes 512-703: nvm command set attributes */ /** submission queue entry size */ - struct { - uint8_t min : 4; - uint8_t max : 4; - } __packed sqes; + uint8_t sqes; /** completion queue entry size */ - struct { - uint8_t min : 4; - uint8_t max : 4; - } __packed cqes; + uint8_t cqes; /** Maximum Outstanding Commands */ uint16_t maxcmd; @@ -658,12 +867,7 @@ struct nvme_controller_data { uint32_t nn; /** optional nvm command support */ - struct { - uint16_t compare : 1; - uint16_t write_unc : 1; - uint16_t dsm: 1; - uint16_t reserved: 13; - } __packed oncs; + uint16_t oncs; /** fused operation support */ uint16_t fuses; @@ -672,16 +876,36 @@ struct nvme_controller_data { uint8_t fna; /** volatile write cache */ - struct { - uint8_t present : 1; - uint8_t reserved : 7; - } __packed vwc; + uint8_t vwc; + + /** Atomic Write Unit Normal */ + uint16_t awun; - /* TODO: flesh out remaining nvm command set attributes */ - uint8_t reserved5[178]; + /** Atomic Write Unit Power Fail */ + uint16_t awupf; - /* bytes 704-2047: i/o command set attributes */ - uint8_t reserved6[1344]; + /** NVM Vendor Specific Command Configuration */ + uint8_t nvscc; + uint8_t reserved5; + + /** Atomic Compare & Write Unit */ + uint16_t acwu; + uint16_t reserved6; + + /** SGL Support */ + uint32_t sgls; + + /* bytes 540-767: Reserved */ + uint8_t reserved7[228]; + + /** NVM Subsystem NVMe Qualified Name */ + uint8_t subnqn[256]; + + /* bytes 1024-1791: Reserved */ + uint8_t reserved8[768]; + + /* bytes 1792-2047: NVMe over Fabrics specification */ + uint8_t reserved9[256]; /* bytes 2048-3071: power state descriptors */ struct nvme_power_state power_state[32]; @@ -704,78 +928,70 @@ struct nvme_namespace_data { uint64_t nuse; /** namespace features */ - struct { - /** thin provisioning */ - uint8_t thin_prov : 1; - uint8_t reserved1 : 7; - } __packed nsfeat; + uint8_t nsfeat; /** number of lba formats */ uint8_t nlbaf; /** formatted lba size */ - struct { - uint8_t format : 4; - uint8_t extended : 1; - uint8_t reserved2 : 3; - } __packed flbas; + uint8_t flbas; /** metadata capabilities */ - struct { - /* metadata can be transferred as part of data prp list */ - uint8_t extended : 1; + uint8_t mc; - /* metadata can be transferred with separate metadata pointer */ - uint8_t pointer : 1; + /** end-to-end data protection capabilities */ + uint8_t dpc; - uint8_t reserved3 : 6; - } __packed mc; + /** end-to-end data protection type settings */ + uint8_t dps; - /** end-to-end data protection capabilities */ - struct { - /* protection information type 1 */ - uint8_t pit1 : 1; + /** Namespace Multi-path I/O and Namespace Sharing Capabilities */ + uint8_t nmic; - /* protection information type 2 */ - uint8_t pit2 : 1; + /** Reservation Capabilities */ + uint8_t rescap; - /* protection information type 3 */ - uint8_t pit3 : 1; + /** Format Progress Indicator */ + uint8_t fpi; - /* first eight bytes of metadata */ - uint8_t md_start : 1; + /** Deallocate Logical Block Features */ + uint8_t dlfeat; - /* last eight bytes of metadata */ - uint8_t md_end : 1; - } __packed dpc; + /** Namespace Atomic Write Unit Normal */ + uint16_t nawun; - /** end-to-end data protection type settings */ - struct { - /* protection information type */ - uint8_t pit : 3; + /** Namespace Atomic Write Unit Power Fail */ + uint16_t nawupf; - /* 1 == protection info transferred at start of metadata */ - /* 0 == protection info transferred at end of metadata */ - uint8_t md_start : 1; + /** Namespace Atomic Compare & Write Unit */ + uint16_t nacwu; - uint8_t reserved4 : 4; - } __packed dps; + /** Namespace Atomic Boundary Size Normal */ + uint16_t nabsn; - uint8_t reserved5[98]; + /** Namespace Atomic Boundary Offset */ + uint16_t nabo; - /** lba format support */ - struct { - /** metadata size */ - uint32_t ms : 16; + /** Namespace Atomic Boundary Size Power Fail */ + uint16_t nabspf; + + /** Namespace Optimal IO Boundary */ + uint16_t noiob; + + /** NVM Capacity */ + uint8_t nvmcap[16]; - /** lba data size */ - uint32_t lbads : 8; + /* bytes 64-103: Reserved */ + uint8_t reserved5[40]; - /** relative performance */ - uint32_t rp : 2; + /** Namespace Globally Unique Identifier */ + uint8_t nguid[16]; - uint32_t reserved6 : 6; - } __packed lbaf[16]; + /** IEEE Extended Unique Identifier */ + uint8_t eui64[8]; + + /** lba format support */ + uint32_t lbaf[16]; uint8_t reserved6[192]; @@ -818,7 +1034,7 @@ struct nvme_error_information_entry { uint64_t error_count; uint16_t sqid; uint16_t cid; - struct nvme_status status; + uint16_t status; uint16_t error_location; uint64_t lba; uint32_t nsid; @@ -828,26 +1044,9 @@ struct nvme_error_information_entry { _Static_assert(sizeof(struct nvme_error_information_entry) == 64, "bad size for nvme_error_information_entry"); -union nvme_critical_warning_state { - - uint8_t raw; - - struct { - uint8_t available_spare : 1; - uint8_t temperature : 1; - uint8_t device_reliability : 1; - uint8_t read_only : 1; - uint8_t volatile_memory_backup : 1; - uint8_t reserved : 3; - } __packed bits; -} __packed; - -_Static_assert(sizeof(union nvme_critical_warning_state) == 1, "bad size for nvme_critical_warning_state"); - struct nvme_health_information_page { - union nvme_critical_warning_state critical_warning; - + uint8_t critical_warning; uint16_t temperature; uint8_t available_spare; uint8_t available_spare_threshold; @@ -884,11 +1083,7 @@ _Static_assert(sizeof(struct nvme_health_information_page) == 512, "bad size for struct nvme_firmware_page { - struct { - uint8_t slot : 3; /* slot for current FW */ - uint8_t reserved : 5; - } __packed afi; - + uint8_t afi; uint8_t reserved[7]; uint64_t revision[7]; /* revisions for 7 slots */ uint8_t reserved2[448]; @@ -987,7 +1182,7 @@ struct nvme_pt_command { }; #define nvme_completion_is_error(cpl) \ - ((cpl)->status.sc != 0 || (cpl)->status.sct != 0) + (NVME_STATUS_GET_SC((cpl)->status) != 0 || NVME_STATUS_GET_SCT((cpl)->status) != 0) void nvme_strvis(uint8_t *dst, const uint8_t *src, int dstlen, int srclen); @@ -1087,19 +1282,19 @@ static inline void nvme_ns_flush_cmd(struct nvme_command *cmd, uint32_t nsid) { - cmd->opc = NVME_OPC_FLUSH; - cmd->nsid = nsid; + cmd->opc_fuse = NVME_CMD_SET_OPC(NVME_OPC_FLUSH); + cmd->nsid = htole32(nsid); } static inline void nvme_ns_rw_cmd(struct nvme_command *cmd, uint32_t rwcmd, uint32_t nsid, uint64_t lba, uint32_t count) { - cmd->opc = rwcmd; - cmd->nsid = nsid; - cmd->cdw10 = lba & 0xffffffffu; - cmd->cdw11 = lba >> 32; - cmd->cdw12 = count-1; + cmd->opc_fuse = NVME_CMD_SET_OPC(rwcmd); + cmd->nsid = htole32(nsid); + cmd->cdw10 = htole32(lba & 0xffffffffu); + cmd->cdw11 = htole32(lba >> 32); + cmd->cdw12 = htole32(count-1); } static inline @@ -1120,14 +1315,173 @@ static inline void nvme_ns_trim_cmd(struct nvme_command *cmd, uint32_t nsid, uint32_t num_ranges) { - cmd->opc = NVME_OPC_DATASET_MANAGEMENT; - cmd->nsid = nsid; - cmd->cdw10 = num_ranges - 1; - cmd->cdw11 = NVME_DSM_ATTR_DEALLOCATE; + cmd->opc_fuse = NVME_CMD_SET_OPC(NVME_OPC_DATASET_MANAGEMENT); + cmd->nsid = htole32(nsid); + cmd->cdw10 = htole32(num_ranges - 1); + cmd->cdw11 = htole32(NVME_DSM_ATTR_DEALLOCATE); } extern int nvme_use_nvd; #endif /* _KERNEL */ +/* Endianess conversion functions for NVMe structs */ +static inline +void nvme_completion_swapbytes(struct nvme_completion *s) +{ + + s->cdw0 = le32toh(s->cdw0); + /* omit rsvd1 */ + s->sqhd = le16toh(s->sqhd); + s->sqid = le16toh(s->sqid); + /* omit cid */ + s->status = le16toh(s->status); +} + +static inline +void nvme_power_state_swapbytes(struct nvme_power_state *s) +{ + + s->mp = le16toh(s->mp); + s->enlat = le32toh(s->enlat); + s->exlat = le32toh(s->exlat); + s->idlp = le16toh(s->idlp); + s->actp = le16toh(s->actp); +} + +static inline +void nvme_controller_data_swapbytes(struct nvme_controller_data *s) +{ + int i; + + s->vid = le16toh(s->vid); + s->ssvid = le16toh(s->ssvid); + s->ctrlr_id = le16toh(s->ctrlr_id); + s->ver = le32toh(s->ver); + s->rtd3r = le32toh(s->rtd3r); + s->rtd3e = le32toh(s->rtd3e); + s->oaes = le32toh(s->oaes); + s->ctratt = le32toh(s->ctratt); + s->oacs = le16toh(s->oacs); + s->wctemp = le16toh(s->wctemp); + s->cctemp = le16toh(s->cctemp); + s->mtfa = le16toh(s->mtfa); + s->hmpre = le32toh(s->hmpre); + s->hmmin = le32toh(s->hmmin); + s->rpmbs = le32toh(s->rpmbs); + s->edstt = le16toh(s->edstt); + s->kas = le16toh(s->kas); + s->hctma = le16toh(s->hctma); + s->mntmt = le16toh(s->mntmt); + s->mxtmt = le16toh(s->mxtmt); + s->sanicap = le32toh(s->sanicap); + s->maxcmd = le16toh(s->maxcmd); + s->nn = le32toh(s->nn); + s->oncs = le16toh(s->oncs); + s->fuses = le16toh(s->fuses); + s->awun = le16toh(s->awun); + s->awupf = le16toh(s->awupf); + s->acwu = le16toh(s->acwu); + s->sgls = le32toh(s->sgls); + for (i = 0; i < 32; i++) + nvme_power_state_swapbytes(&s->power_state[i]); +} + +static inline +void nvme_namespace_data_swapbytes(struct nvme_namespace_data *s) +{ + int i; + + s->nsze = le64toh(s->nsze); + s->ncap = le64toh(s->ncap); + s->nuse = le64toh(s->nuse); + s->nawun = le16toh(s->nawun); + s->nawupf = le16toh(s->nawupf); + s->nacwu = le16toh(s->nacwu); + s->nabsn = le16toh(s->nabsn); + s->nabo = le16toh(s->nabo); + s->nabspf = le16toh(s->nabspf); + s->noiob = le16toh(s->noiob); + for (i = 0; i < 16; i++) + s->lbaf[i] = le32toh(s->lbaf[i]); +} + +static inline +void nvme_error_information_entry_swapbytes(struct nvme_error_information_entry *s) +{ + + s->error_count = le64toh(s->error_count); + s->sqid = le16toh(s->sqid); + s->cid = le16toh(s->cid); + s->status = le16toh(s->status); + s->error_location = le16toh(s->error_location); + s->lba = le64toh(s->lba); + s->nsid = le32toh(s->nsid); +} + +static inline +void nvme_le128toh(void *p) +{ +#if _BYTE_ORDER != _LITTLE_ENDIAN + /* Swap 16 bytes in place */ + char *tmp = (char*)p; + char b; + int i; + for (i = 0; i < 8; i++) { + b = tmp[i]; + tmp[i] = tmp[15-i]; + tmp[15-i] = b; + } +#else + (void)p; +#endif +} + +static inline +void nvme_health_information_page_swapbytes(struct nvme_health_information_page *s) +{ + int i; + + s->temperature = le16toh(s->temperature); + nvme_le128toh((void *)s->data_units_read); + nvme_le128toh((void *)s->data_units_written); + nvme_le128toh((void *)s->host_read_commands); + nvme_le128toh((void *)s->host_write_commands); + nvme_le128toh((void *)s->controller_busy_time); + nvme_le128toh((void *)s->power_cycles); + nvme_le128toh((void *)s->power_on_hours); + nvme_le128toh((void *)s->unsafe_shutdowns); + nvme_le128toh((void *)s->media_errors); + nvme_le128toh((void *)s->num_error_info_log_entries); + s->warning_temp_time = le32toh(s->warning_temp_time); + s->error_temp_time = le32toh(s->error_temp_time); + for (i = 0; i < 8; i++) + s->temp_sensor[i] = le16toh(s->temp_sensor[i]); +} + + +static inline +void nvme_firmware_page_swapbytes(struct nvme_firmware_page *s) +{ + int i; + + for (i = 0; i < 7; i++) + s->revision[i] = le64toh(s->revision[i]); +} + +static inline +void intel_log_temp_stats_swapbytes(struct intel_log_temp_stats *s) +{ + + s->current = le64toh(s->current); + s->overtemp_flag_last = le64toh(s->overtemp_flag_last); + s->overtemp_flag_life = le64toh(s->overtemp_flag_life); + s->max_temp = le64toh(s->max_temp); + s->min_temp = le64toh(s->min_temp); + /* omit _rsvd[] */ + s->max_oper_temp = le64toh(s->max_oper_temp); + s->min_oper_temp = le64toh(s->min_oper_temp); + s->est_offset = le64toh(s->est_offset); +} + #endif /* __NVME_H__ */ diff --git a/freebsd/sys/dev/ofw/ofw_fdt.c b/freebsd/sys/dev/ofw/ofw_fdt.c index b9266eb5..8878b5c9 100644 --- a/freebsd/sys/dev/ofw/ofw_fdt.c +++ b/freebsd/sys/dev/ofw/ofw_fdt.c @@ -182,7 +182,7 @@ fdt_phandle_offset(phandle_t p) static phandle_t ofw_fdt_peer(ofw_t ofw, phandle_t node) { - int depth, offset; + int offset; if (node == 0) { /* Find root node */ @@ -194,39 +194,21 @@ ofw_fdt_peer(ofw_t ofw, phandle_t node) offset = fdt_phandle_offset(node); if (offset < 0) return (0); - - for (depth = 1, offset = fdt_next_node(fdtp, offset, &depth); - offset >= 0; - offset = fdt_next_node(fdtp, offset, &depth)) { - if (depth < 0) - return (0); - if (depth == 1) - return (fdt_offset_phandle(offset)); - } - - return (0); + offset = fdt_next_subnode(fdtp, offset); + return (fdt_offset_phandle(offset)); } /* Return the first child of this node or 0. */ static phandle_t ofw_fdt_child(ofw_t ofw, phandle_t node) { - int depth, offset; + int offset; offset = fdt_phandle_offset(node); if (offset < 0) return (0); - - for (depth = 0, offset = fdt_next_node(fdtp, offset, &depth); - (offset >= 0) && (depth > 0); - offset = fdt_next_node(fdtp, offset, &depth)) { - if (depth < 0) - return (0); - if (depth == 1) - return (fdt_offset_phandle(offset)); - } - - return (0); + offset = fdt_first_subnode(fdtp, offset); + return (fdt_offset_phandle(offset)); } /* Return the parent of this node or 0. */ @@ -349,27 +331,25 @@ ofw_fdt_nextprop(ofw_t ofw, phandle_t package, const char *previous, char *buf, if (offset < 0) return (-1); - /* Find the first prop in the node */ - offset = fdt_first_property_offset(fdtp, offset); - if (offset < 0) - return (0); /* No properties */ - - if (previous != NULL) { - while (offset >= 0) { + if (previous == NULL) + /* Find the first prop in the node */ + offset = fdt_first_property_offset(fdtp, offset); + else { + fdt_for_each_property_offset(offset, fdtp, offset) { prop = fdt_getprop_by_offset(fdtp, offset, &name, NULL); if (prop == NULL) return (-1); /* Internal error */ - + /* Skip until we find 'previous', then bail out */ + if (strcmp(name, previous) != 0) + continue; offset = fdt_next_property_offset(fdtp, offset); - if (offset < 0) - return (0); /* No more properties */ - - /* Check if the last one was the one we wanted */ - if (strcmp(name, previous) == 0) - break; + break; } } + if (offset < 0) + return (0); /* No properties */ + prop = fdt_getprop_by_offset(fdtp, offset, &name, &offset); if (prop == NULL) return (-1); /* Internal error */ diff --git a/freebsd/sys/dev/ofw/ofw_subr.c b/freebsd/sys/dev/ofw/ofw_subr.c index e2528691..8359485e 100644 --- a/freebsd/sys/dev/ofw/ofw_subr.c +++ b/freebsd/sys/dev/ofw/ofw_subr.c @@ -234,7 +234,7 @@ ofw_parse_bootargs(void) int err; chosen = OF_finddevice("/chosen"); - if (chosen <= 0) + if (chosen == -1) return (chosen); if ((err = OF_getprop(chosen, "bootargs", buf, sizeof(buf))) != -1) { diff --git a/freebsd/sys/dev/pci/pci.c b/freebsd/sys/dev/pci/pci.c index 7fbdc693..55a21320 100644 --- a/freebsd/sys/dev/pci/pci.c +++ b/freebsd/sys/dev/pci/pci.c @@ -185,8 +185,11 @@ static device_method_t pci_methods[] = { DEVMETHOD(pci_set_powerstate, pci_set_powerstate_method), DEVMETHOD(pci_assign_interrupt, pci_assign_interrupt_method), DEVMETHOD(pci_find_cap, pci_find_cap_method), + DEVMETHOD(pci_find_next_cap, pci_find_next_cap_method), DEVMETHOD(pci_find_extcap, pci_find_extcap_method), + DEVMETHOD(pci_find_next_extcap, pci_find_next_extcap_method), DEVMETHOD(pci_find_htcap, pci_find_htcap_method), + DEVMETHOD(pci_find_next_htcap, pci_find_next_htcap_method), DEVMETHOD(pci_alloc_msi, pci_alloc_msi_method), DEVMETHOD(pci_alloc_msix, pci_alloc_msix_method), DEVMETHOD(pci_enable_msi, pci_enable_msi_method), @@ -1381,7 +1384,7 @@ pci_find_htcap_method(device_t dev, device_t child, int capability, int *capreg) * Traverse the capabilities list checking each HT capability * to see if it matches the requested HT capability. */ - while (ptr != 0) { + for (;;) { val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2); if (capability == PCIM_HTCAP_SLAVE || capability == PCIM_HTCAP_HOST) @@ -1395,13 +1398,51 @@ pci_find_htcap_method(device_t dev, device_t child, int capability, int *capreg) } /* Skip to the next HT capability. */ - while (ptr != 0) { - ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1); - if (pci_read_config(child, ptr + PCICAP_ID, 1) == - PCIY_HT) - break; + if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0) + break; + } + + return (ENOENT); +} + +/* + * Find the next requested HyperTransport capability after start and return + * the offset in configuration space via the pointer provided. The function + * returns 0 on success and an error code otherwise. + */ +int +pci_find_next_htcap_method(device_t dev, device_t child, int capability, + int start, int *capreg) +{ + int ptr; + uint16_t val; + + KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == PCIY_HT, + ("start capability is not HyperTransport capability")); + ptr = start; + + /* + * Traverse the capabilities list checking each HT capability + * to see if it matches the requested HT capability. + */ + for (;;) { + /* Skip to the next HT capability. */ + if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0) + break; + + val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2); + if (capability == PCIM_HTCAP_SLAVE || + capability == PCIM_HTCAP_HOST) + val &= 0xe000; + else + val &= PCIM_HTCMD_CAP_MASK; + if (val == capability) { + if (capreg != NULL) + *capreg = ptr; + return (0); } } + return (ENOENT); } @@ -1416,8 +1457,8 @@ pci_find_cap_method(device_t dev, device_t child, int capability, { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; - u_int32_t status; - u_int8_t ptr; + uint32_t status; + uint8_t ptr; /* * Check the CAP_LIST bit of the PCI status register first. @@ -1459,6 +1500,33 @@ pci_find_cap_method(device_t dev, device_t child, int capability, } /* + * Find the next requested capability after start and return the offset in + * configuration space via the pointer provided. The function returns + * 0 on success and an error code otherwise. + */ +int +pci_find_next_cap_method(device_t dev, device_t child, int capability, + int start, int *capreg) +{ + uint8_t ptr; + + KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == capability, + ("start capability is not expected capability")); + + ptr = pci_read_config(child, start + PCICAP_NEXTPTR, 1); + while (ptr != 0) { + if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) { + if (capreg != NULL) + *capreg = ptr; + return (0); + } + ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1); + } + + return (ENOENT); +} + +/* * Find the requested extended capability and return the offset in * configuration space via the pointer provided. The function returns * 0 on success and an error code otherwise. @@ -1496,6 +1564,41 @@ pci_find_extcap_method(device_t dev, device_t child, int capability, } /* + * Find the next requested extended capability after start and return the + * offset in configuration space via the pointer provided. The function + * returns 0 on success and an error code otherwise. + */ +int +pci_find_next_extcap_method(device_t dev, device_t child, int capability, + int start, int *capreg) +{ + struct pci_devinfo *dinfo = device_get_ivars(child); + pcicfgregs *cfg = &dinfo->cfg; + uint32_t ecap; + uint16_t ptr; + + /* Only supported for PCI-express devices. */ + if (cfg->pcie.pcie_location == 0) + return (ENXIO); + + ecap = pci_read_config(child, start, 4); + KASSERT(PCI_EXTCAP_ID(ecap) == capability, + ("start extended capability is not expected capability")); + ptr = PCI_EXTCAP_NEXTPTR(ecap); + while (ptr != 0) { + ecap = pci_read_config(child, ptr, 4); + if (PCI_EXTCAP_ID(ecap) == capability) { + if (capreg != NULL) + *capreg = ptr; + return (0); + } + ptr = PCI_EXTCAP_NEXTPTR(ecap); + } + + return (ENOENT); +} + +/* * Support for MSI-X message interrupts. */ static void diff --git a/freebsd/sys/dev/pci/pci_private.h b/freebsd/sys/dev/pci/pci_private.h index dd254e66..f468152b 100644 --- a/freebsd/sys/dev/pci/pci_private.h +++ b/freebsd/sys/dev/pci/pci_private.h @@ -90,10 +90,16 @@ int pci_enable_io_method(device_t dev, device_t child, int space); int pci_disable_io_method(device_t dev, device_t child, int space); int pci_find_cap_method(device_t dev, device_t child, int capability, int *capreg); +int pci_find_next_cap_method(device_t dev, device_t child, + int capability, int start, int *capreg); int pci_find_extcap_method(device_t dev, device_t child, int capability, int *capreg); +int pci_find_next_extcap_method(device_t dev, device_t child, + int capability, int start, int *capreg); int pci_find_htcap_method(device_t dev, device_t child, int capability, int *capreg); +int pci_find_next_htcap_method(device_t dev, device_t child, + int capability, int start, int *capreg); int pci_alloc_msi_method(device_t dev, device_t child, int *count); int pci_alloc_msix_method(device_t dev, device_t child, int *count); void pci_enable_msi_method(device_t dev, device_t child, diff --git a/freebsd/sys/dev/pci/pci_user.c b/freebsd/sys/dev/pci/pci_user.c index 6e1cc3ef..e6297bd7 100644 --- a/freebsd/sys/dev/pci/pci_user.c +++ b/freebsd/sys/dev/pci/pci_user.c @@ -764,12 +764,16 @@ pci_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *t * tell the user that there are more matches * left. */ - if (cio->num_matches >= ionum) + if (cio->num_matches >= ionum) { + error = 0; break; + } #ifdef PRE7_COMPAT #ifdef COMPAT_FREEBSD32 if (cmd == PCIOCGETCONF_OLD32) { + memset(&conf_old32, 0, + sizeof(conf_old32)); conf_old32.pc_sel.pc_bus = dinfo->conf.pc_sel.pc_bus; conf_old32.pc_sel.pc_dev = @@ -803,6 +807,7 @@ pci_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *t } else #endif /* COMPAT_FREEBSD32 */ if (cmd == PCIOCGETCONF_OLD) { + memset(&conf_old, 0, sizeof(conf_old)); conf_old.pc_sel.pc_bus = dinfo->conf.pc_sel.pc_bus; conf_old.pc_sel.pc_dev = diff --git a/freebsd/sys/dev/pci/pcivar.h b/freebsd/sys/dev/pci/pcivar.h index 430e29db..492b7117 100644 --- a/freebsd/sys/dev/pci/pcivar.h +++ b/freebsd/sys/dev/pci/pcivar.h @@ -468,18 +468,39 @@ pci_find_cap(device_t dev, int capability, int *capreg) } static __inline int +pci_find_next_cap(device_t dev, int capability, int start, int *capreg) +{ + return (PCI_FIND_NEXT_CAP(device_get_parent(dev), dev, capability, start, + capreg)); +} + +static __inline int pci_find_extcap(device_t dev, int capability, int *capreg) { return (PCI_FIND_EXTCAP(device_get_parent(dev), dev, capability, capreg)); } static __inline int +pci_find_next_extcap(device_t dev, int capability, int start, int *capreg) +{ + return (PCI_FIND_NEXT_EXTCAP(device_get_parent(dev), dev, capability, + start, capreg)); +} + +static __inline int pci_find_htcap(device_t dev, int capability, int *capreg) { return (PCI_FIND_HTCAP(device_get_parent(dev), dev, capability, capreg)); } static __inline int +pci_find_next_htcap(device_t dev, int capability, int start, int *capreg) +{ + return (PCI_FIND_NEXT_HTCAP(device_get_parent(dev), dev, capability, + start, capreg)); +} + +static __inline int pci_alloc_msi(device_t dev, int *count) { return (PCI_ALLOC_MSI(device_get_parent(dev), dev, count)); diff --git a/freebsd/sys/dev/rtwn/if_rtwn_ridx.h b/freebsd/sys/dev/rtwn/if_rtwn_ridx.h index 76f87aab..2e6f8d98 100644 --- a/freebsd/sys/dev/rtwn/if_rtwn_ridx.h +++ b/freebsd/sys/dev/rtwn/if_rtwn_ridx.h @@ -36,7 +36,9 @@ #define RTWN_RIDX_OFDM36 9 #define RTWN_RIDX_OFDM48 10 #define RTWN_RIDX_OFDM54 11 -#define RTWN_RIDX_HT_MCS(i) (12 + (i)) + +#define RTWN_RIDX_HT_MCS_SHIFT 12 +#define RTWN_RIDX_HT_MCS(i) (RTWN_RIDX_HT_MCS_SHIFT + (i)) #define RTWN_RIDX_COUNT 28 #define RTWN_RIDX_UNKNOWN (uint8_t)-1 @@ -53,8 +55,7 @@ static __inline uint8_t rate2ridx(uint8_t rate) { if (rate & IEEE80211_RATE_MCS) { - /* 11n rates start at idx 12 */ - return ((rate & 0xf) + 12); + return ((rate & 0xf) + RTWN_RIDX_HT_MCS_SHIFT); } switch (rate) { /* 11g */ diff --git a/freebsd/sys/dev/rtwn/rtl8188e/r88e_chan.c b/freebsd/sys/dev/rtwn/rtl8188e/r88e_chan.c index 530df320..e57364ac 100644 --- a/freebsd/sys/dev/rtwn/rtl8188e/r88e_chan.c +++ b/freebsd/sys/dev/rtwn/rtl8188e/r88e_chan.c @@ -106,8 +106,6 @@ r88e_get_txpower(struct rtwn_softc *sc, int chain, max_mcs = RTWN_RIDX_HT_MCS(sc->ntxchains * 8 - 1); KASSERT(max_mcs <= RTWN_RIDX_COUNT, ("increase ridx limit\n")); - memset(power, 0, max_mcs * sizeof(power[0])); - /* Compute per-CCK rate Tx power. */ cckpow = rt->cck_tx_pwr[group]; for (ridx = RTWN_RIDX_CCK1; ridx <= RTWN_RIDX_CCK11; ridx++) { diff --git a/freebsd/sys/dev/rtwn/rtl8188e/r88e_rx.c b/freebsd/sys/dev/rtwn/rtl8188e/r88e_rx.c index 856ec88b..53cc722f 100644 --- a/freebsd/sys/dev/rtwn/rtl8188e/r88e_rx.c +++ b/freebsd/sys/dev/rtwn/rtl8188e/r88e_rx.c @@ -110,7 +110,8 @@ r88e_ratectl_tx_complete(struct rtwn_softc *sc, uint8_t *buf, int len) txs.long_retries = ntries; if (rpt->final_rate > RTWN_RIDX_OFDM54) { /* MCS */ txs.final_rate = - (rpt->final_rate - 12) | IEEE80211_RATE_MCS; + rpt->final_rate - RTWN_RIDX_HT_MCS_SHIFT; + txs.final_rate |= IEEE80211_RATE_MCS; } else txs.final_rate = ridx2rate[rpt->final_rate]; if (rpt->rptb1 & R88E_RPTB1_PKT_OK) diff --git a/freebsd/sys/dev/rtwn/rtl8192c/r92c_chan.c b/freebsd/sys/dev/rtwn/rtl8192c/r92c_chan.c index 2e50daf1..dd1ff6de 100644 --- a/freebsd/sys/dev/rtwn/rtl8192c/r92c_chan.c +++ b/freebsd/sys/dev/rtwn/rtl8192c/r92c_chan.c @@ -104,7 +104,6 @@ r92c_get_txpower(struct rtwn_softc *sc, int chain, max_mcs = RTWN_RIDX_HT_MCS(sc->ntxchains * 8 - 1); KASSERT(max_mcs <= RTWN_RIDX_COUNT, ("increase ridx limit\n")); - memset(power, 0, max_mcs * sizeof(power[0])); if (rs->regulatory == 0) { for (ridx = RTWN_RIDX_CCK1; ridx <= RTWN_RIDX_CCK11; ridx++) power[ridx] = base[chain].pwr[0][ridx]; @@ -227,6 +226,7 @@ r92c_set_txpower(struct rtwn_softc *sc, struct ieee80211_channel *c) int i; for (i = 0; i < sc->ntxchains; i++) { + memset(power, 0, sizeof(power)); /* Compute per-rate Tx power values. */ rtwn_r92c_get_txpower(sc, i, c, power); #ifdef RTWN_DEBUG diff --git a/freebsd/sys/dev/rtwn/rtl8192c/r92c_rx.c b/freebsd/sys/dev/rtwn/rtl8192c/r92c_rx.c index 80d2859c..c98a0203 100644 --- a/freebsd/sys/dev/rtwn/rtl8192c/r92c_rx.c +++ b/freebsd/sys/dev/rtwn/rtl8192c/r92c_rx.c @@ -142,7 +142,8 @@ r92c_get_rx_stats(struct rtwn_softc *sc, struct ieee80211_rx_stats *rxs, else rxs->c_pktflags |= IEEE80211_RX_F_OFDM; } else { /* MCS0~15. */ - rxs->c_rate = IEEE80211_RATE_MCS | (rate - 12); + rxs->c_rate = + IEEE80211_RATE_MCS | (rate - RTWN_RIDX_HT_MCS_SHIFT); rxs->c_pktflags |= IEEE80211_RX_F_HT; } } diff --git a/freebsd/sys/dev/rtwn/rtl8812a/r12a_rx.c b/freebsd/sys/dev/rtwn/rtl8812a/r12a_rx.c index c2dc4c3a..a59c191c 100644 --- a/freebsd/sys/dev/rtwn/rtl8812a/r12a_rx.c +++ b/freebsd/sys/dev/rtwn/rtl8812a/r12a_rx.c @@ -117,7 +117,8 @@ r12a_ratectl_tx_complete(struct rtwn_softc *sc, uint8_t *buf, int len) txs.long_retries = ntries; if (rpt->final_rate > RTWN_RIDX_OFDM54) { /* MCS */ txs.final_rate = - (rpt->final_rate - 12) | IEEE80211_RATE_MCS; + rpt->final_rate - RTWN_RIDX_HT_MCS_SHIFT; + txs.final_rate |= IEEE80211_RATE_MCS; } else txs.final_rate = ridx2rate[rpt->final_rate]; if (rpt->txrptb0 & R12A_TXRPTB0_RETRY_OVER) @@ -312,7 +313,8 @@ r12a_get_rx_stats(struct rtwn_softc *sc, struct ieee80211_rx_stats *rxs, rxs->c_pktflags |= IEEE80211_RX_F_OFDM; } else { /* MCS0~15. */ /* TODO: VHT rates */ - rxs->c_rate = IEEE80211_RATE_MCS | (rate - 12); + rxs->c_rate = + IEEE80211_RATE_MCS | (rate - RTWN_RIDX_HT_MCS_SHIFT); rxs->c_pktflags |= IEEE80211_RX_F_HT; } diff --git a/freebsd/sys/dev/usb/controller/musb_otg.c b/freebsd/sys/dev/usb/controller/musb_otg.c index 16a3adef..99290386 100644 --- a/freebsd/sys/dev/usb/controller/musb_otg.c +++ b/freebsd/sys/dev/usb/controller/musb_otg.c @@ -151,6 +151,27 @@ static const struct usb_hw_ep_profile musbotg_ep_profile[1] = { } }; +static const struct musb_otg_ep_cfg musbotg_ep_default[] = { + { + .ep_end = 1, + .ep_fifosz_shift = 12, + .ep_fifosz_reg = MUSB2_VAL_FIFOSZ_4096 | MUSB2_MASK_FIFODB, + }, + { + .ep_end = 7, + .ep_fifosz_shift = 10, + .ep_fifosz_reg = MUSB2_VAL_FIFOSZ_512 | MUSB2_MASK_FIFODB, + }, + { + .ep_end = 15, + .ep_fifosz_shift = 7, + .ep_fifosz_reg = MUSB2_VAL_FIFOSZ_128, + }, + { + .ep_end = -1, + }, +}; + static int musbotg_channel_alloc(struct musbotg_softc *sc, struct musbotg_td *td, uint8_t is_tx) { @@ -3061,7 +3082,9 @@ musbotg_clear_stall(struct usb_device *udev, struct usb_endpoint *ep) usb_error_t musbotg_init(struct musbotg_softc *sc) { + const struct musb_otg_ep_cfg *cfg; struct usb_hw_ep_profile *pf; + int i; uint16_t offset; uint8_t nrx; uint8_t ntx; @@ -3077,6 +3100,10 @@ musbotg_init(struct musbotg_softc *sc) sc->sc_bus.usbrev = USB_REV_2_0; sc->sc_bus.methods = &musbotg_bus_methods; + /* Set a default endpoint configuration */ + if (sc->sc_ep_cfg == NULL) + sc->sc_ep_cfg = musbotg_ep_default; + USB_BUS_LOCK(&sc->sc_bus); /* turn on clocks */ @@ -3143,19 +3170,24 @@ musbotg_init(struct musbotg_softc *sc) MUSB2_WRITE_1(sc, MUSB2_REG_EPINDEX, 0); - /* read out number of endpoints */ + if (sc->sc_ep_max == 0) { + /* read out number of endpoints */ + + nrx = + (MUSB2_READ_1(sc, MUSB2_REG_EPINFO) / 16); - nrx = - (MUSB2_READ_1(sc, MUSB2_REG_EPINFO) / 16); + ntx = + (MUSB2_READ_1(sc, MUSB2_REG_EPINFO) % 16); - ntx = - (MUSB2_READ_1(sc, MUSB2_REG_EPINFO) % 16); + sc->sc_ep_max = (nrx > ntx) ? nrx : ntx; + } else { + nrx = ntx = sc->sc_ep_max; + } /* these numbers exclude the control endpoint */ DPRINTFN(2, "RX/TX endpoints: %u/%u\n", nrx, ntx); - sc->sc_ep_max = (nrx > ntx) ? nrx : ntx; if (sc->sc_ep_max == 0) { DPRINTFN(2, "ERROR: Looks like the clocks are off!\n"); } @@ -3195,20 +3227,15 @@ musbotg_init(struct musbotg_softc *sc) if (dynfifo) { if (frx && (temp <= nrx)) { - if (temp == 1) { - frx = 12; /* 4K */ - MUSB2_WRITE_1(sc, MUSB2_REG_RXFIFOSZ, - MUSB2_VAL_FIFOSZ_4096 | - MUSB2_MASK_FIFODB); - } else if (temp < 8) { - frx = 10; /* 1K */ - MUSB2_WRITE_1(sc, MUSB2_REG_RXFIFOSZ, - MUSB2_VAL_FIFOSZ_512 | - MUSB2_MASK_FIFODB); - } else { - frx = 7; /* 128 bytes */ - MUSB2_WRITE_1(sc, MUSB2_REG_RXFIFOSZ, - MUSB2_VAL_FIFOSZ_128); + for (i = 0; sc->sc_ep_cfg[i].ep_end >= 0; i++) { + cfg = &sc->sc_ep_cfg[i]; + if (temp <= cfg->ep_end) { + frx = cfg->ep_fifosz_shift; + MUSB2_WRITE_1(sc, + MUSB2_REG_RXFIFOSZ, + cfg->ep_fifosz_reg); + break; + } } MUSB2_WRITE_2(sc, MUSB2_REG_RXFIFOADD, @@ -3217,20 +3244,15 @@ musbotg_init(struct musbotg_softc *sc) offset += (1 << frx); } if (ftx && (temp <= ntx)) { - if (temp == 1) { - ftx = 12; /* 4K */ - MUSB2_WRITE_1(sc, MUSB2_REG_TXFIFOSZ, - MUSB2_VAL_FIFOSZ_4096 | - MUSB2_MASK_FIFODB); - } else if (temp < 8) { - ftx = 10; /* 1K */ - MUSB2_WRITE_1(sc, MUSB2_REG_TXFIFOSZ, - MUSB2_VAL_FIFOSZ_512 | - MUSB2_MASK_FIFODB); - } else { - ftx = 7; /* 128 bytes */ - MUSB2_WRITE_1(sc, MUSB2_REG_TXFIFOSZ, - MUSB2_VAL_FIFOSZ_128); + for (i = 0; sc->sc_ep_cfg[i].ep_end >= 0; i++) { + cfg = &sc->sc_ep_cfg[i]; + if (temp <= cfg->ep_end) { + ftx = cfg->ep_fifosz_shift; + MUSB2_WRITE_1(sc, + MUSB2_REG_TXFIFOSZ, + cfg->ep_fifosz_reg); + break; + } } MUSB2_WRITE_2(sc, MUSB2_REG_TXFIFOADD, diff --git a/freebsd/sys/dev/usb/controller/musb_otg.h b/freebsd/sys/dev/usb/controller/musb_otg.h index 55ab7aa6..5d445a3d 100644 --- a/freebsd/sys/dev/usb/controller/musb_otg.h +++ b/freebsd/sys/dev/usb/controller/musb_otg.h @@ -387,6 +387,12 @@ struct musbotg_flags { uint8_t d_pulled_up:1; }; +struct musb_otg_ep_cfg { + int ep_end; + int ep_fifosz_shift; + uint8_t ep_fifosz_reg; +}; + struct musbotg_softc { struct usb_bus sc_bus; union musbotg_hub_temp sc_hub_temp; @@ -423,6 +429,7 @@ struct musbotg_softc { uint8_t sc_id; uint8_t sc_mode; void *sc_platform_data; + const struct musb_otg_ep_cfg *sc_ep_cfg; }; /* prototypes */ diff --git a/freebsd/sys/dev/usb/input/ukbd.c b/freebsd/sys/dev/usb/input/ukbd.c index 9b1d28fc..5cb2636c 100644 --- a/freebsd/sys/dev/usb/input/ukbd.c +++ b/freebsd/sys/dev/usb/input/ukbd.c @@ -388,10 +388,22 @@ ukbd_any_key_pressed(struct ukbd_softc *sc) static void ukbd_start_timer(struct ukbd_softc *sc) { - sbintime_t delay, prec; + sbintime_t delay, now, prec; + now = sbinuptime(); + + /* check if initial delay passed and fallback to key repeat delay */ + if (sc->sc_delay == 0) + sc->sc_delay = sc->sc_kbd.kb_delay2; + + /* compute timeout */ delay = SBT_1MS * sc->sc_delay; sc->sc_co_basetime += delay; + + /* check if we are running behind */ + if (sc->sc_co_basetime < now) + sc->sc_co_basetime = now; + /* This is rarely called, so prefer precision to efficiency. */ prec = qmin(delay >> 7, SBT_1MS * 10); usb_callout_reset_sbt(&sc->sc_callout, sc->sc_co_basetime, prec, @@ -512,7 +524,6 @@ ukbd_get_key(struct ukbd_softc *sc, uint8_t wait) static void ukbd_interrupt(struct ukbd_softc *sc) { - struct timeval ctv; uint32_t n_mod; uint32_t o_mod; uint32_t now = sc->sc_time_ms; @@ -582,14 +593,11 @@ rfound: ; break; } } - if (j < UKBD_NKEYCODE) { - /* Old key repeating. */ - sc->sc_delay = sc->sc_kbd.kb_delay2; - } else { - /* New key. */ - microuptime(&ctv); - sc->sc_co_basetime = tvtosbt(ctv); + if (j == UKBD_NKEYCODE) { + /* New key - set initial delay and [re]start timer */ + sc->sc_co_basetime = sbinuptime(); sc->sc_delay = sc->sc_kbd.kb_delay1; + ukbd_start_timer(sc); } ukbd_put_key(sc, key | KEY_PRESS); @@ -840,10 +848,6 @@ ukbd_intr_callback(struct usb_xfer *xfer, usb_error_t error) ukbd_interrupt(sc); - if (ukbd_any_key_pressed(sc) != 0) { - ukbd_start_timer(sc); - } - case USB_ST_SETUP: tr_setup: if (sc->sc_inputs < UKBD_IN_BUF_FULL) { diff --git a/freebsd/sys/dev/usb/input/wsp.c b/freebsd/sys/dev/usb/input/wsp.c index 0eb2c024..02fa7722 100644 --- a/freebsd/sys/dev/usb/input/wsp.c +++ b/freebsd/sys/dev/usb/input/wsp.c @@ -931,7 +931,12 @@ wsp_intr_callback(struct usb_xfer *xfer, usb_error_t error) sc->sc_status.button = 0; if (ibt != 0) { - sc->sc_status.button |= MOUSE_BUTTON1DOWN; + if ((params->caps & HAS_INTEGRATED_BUTTON) && ntouch == 2) + sc->sc_status.button |= MOUSE_BUTTON3DOWN; + else if ((params->caps & HAS_INTEGRATED_BUTTON) && ntouch == 3) + sc->sc_status.button |= MOUSE_BUTTON2DOWN; + else + sc->sc_status.button |= MOUSE_BUTTON1DOWN; sc->ibtn = 1; } sc->intr_count++; diff --git a/freebsd/sys/dev/usb/net/if_axge.c b/freebsd/sys/dev/usb/net/if_axge.c index fc602b88..575571e4 100644 --- a/freebsd/sys/dev/usb/net/if_axge.c +++ b/freebsd/sys/dev/usb/net/if_axge.c @@ -589,6 +589,10 @@ axge_detach(device_t dev) sc = device_get_softc(dev); ue = &sc->sc_ue; if (device_is_attached(dev)) { + + /* wait for any post attach or other command to complete */ + usb_proc_drain(&ue->ue_tq); + AXGE_LOCK(sc); /* * XXX @@ -880,7 +884,8 @@ axge_stop(struct usb_ether *ue) val &= ~MSR_RE; axge_write_cmd_2(sc, AXGE_ACCESS_MAC, 2, AXGE_MSR, val); - ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); + if (ifp != NULL) + ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); sc->sc_flags &= ~AXGE_FLAG_LINK; /* diff --git a/freebsd/sys/dev/usb/quirk/usb_quirk.c b/freebsd/sys/dev/usb/quirk/usb_quirk.c index 3ae4a557..2aea57c8 100644 --- a/freebsd/sys/dev/usb/quirk/usb_quirk.c +++ b/freebsd/sys/dev/usb/quirk/usb_quirk.c @@ -139,6 +139,8 @@ static struct usb_quirk_entry usb_quirks[USB_DEV_QUIRKS_MAX] = { USB_QUIRK(CORSAIR, K60, 0x0000, 0xffff, UQ_KBD_BOOTPROTO), /* Quirk for Corsair Vengeance K70 keyboard */ USB_QUIRK(CORSAIR, K70, 0x0000, 0xffff, UQ_KBD_BOOTPROTO), + /* Quirk for Corsair K70 RGB keyboard */ + USB_QUIRK(CORSAIR, K70_RGB, 0x0000, 0xffff, UQ_KBD_BOOTPROTO), /* Quirk for Corsair STRAFE Gaming keyboard */ USB_QUIRK(CORSAIR, STRAFE, 0x0000, 0xffff, UQ_KBD_BOOTPROTO), /* umodem(4) device quirks */ diff --git a/freebsd/sys/dev/usb/serial/uftdi.c b/freebsd/sys/dev/usb/serial/uftdi.c index d07c055d..9f20662c 100644 --- a/freebsd/sys/dev/usb/serial/uftdi.c +++ b/freebsd/sys/dev/usb/serial/uftdi.c @@ -443,7 +443,7 @@ static const STRUCT_USB_HOST_ID uftdi_devs[] = { UFTDI_DEV(FTDI, IPLUS2, 0), UFTDI_DEV(FTDI, IRTRANS, 0), UFTDI_DEV(FTDI, KBS, 0), - UFTDI_DEV(FTDI, KTLINK, 0), + UFTDI_DEV(FTDI, KTLINK, UFTDI_JTAG_IFACE(0)), UFTDI_DEV(FTDI, LENZ_LIUSB, 0), UFTDI_DEV(FTDI, LK202, 0), UFTDI_DEV(FTDI, LK204, 0), @@ -524,6 +524,7 @@ static const STRUCT_USB_HOST_ID uftdi_devs[] = { UFTDI_DEV(FTDI, TERATRONIK_D2XX, 0), UFTDI_DEV(FTDI, TERATRONIK_VCP, 0), UFTDI_DEV(FTDI, THORLABS, 0), + UFTDI_DEV(FTDI, TIAO, 0), UFTDI_DEV(FTDI, TNC_X, 0), UFTDI_DEV(FTDI, TTUSB, 0), UFTDI_DEV(FTDI, TURTELIZER2, UFTDI_JTAG_IFACE(0)), diff --git a/freebsd/sys/dev/usb/serial/uslcom.c b/freebsd/sys/dev/usb/serial/uslcom.c index 08a563de..45835b82 100644 --- a/freebsd/sys/dev/usb/serial/uslcom.c +++ b/freebsd/sys/dev/usb/serial/uslcom.c @@ -315,6 +315,7 @@ static const STRUCT_USB_HOST_ID uslcom_devs[] = { USLCOM_DEV(SILABS, EMS_C1007), USLCOM_DEV(SILABS, HAMLINKUSB), USLCOM_DEV(SILABS, HELICOM), + USLCOM_DEV(SILABS, HUBZ), USLCOM_DEV(SILABS, IMS_USB_RS422), USLCOM_DEV(SILABS, INFINITY_MIC), USLCOM_DEV(SILABS, INGENI_ZIGBEE), diff --git a/freebsd/sys/dev/usb/usb_lookup.c b/freebsd/sys/dev/usb/usb_lookup.c index 6f6a763f..28b01afa 100644 --- a/freebsd/sys/dev/usb/usb_lookup.c +++ b/freebsd/sys/dev/usb/usb_lookup.c @@ -155,70 +155,3 @@ usbd_lookup_id_by_uaa(const struct usb_device_id *id, usb_size_t sizeof_id, return (ENXIO); } -/*------------------------------------------------------------------------* - * Export the USB device ID format we use to userspace tools. - *------------------------------------------------------------------------*/ -#if BYTE_ORDER == LITTLE_ENDIAN -#define U16_XOR "0" -#else -#define U16_XOR "8" -#endif - -#if defined(KLD_MODULE) && (USB_HAVE_ID_SECTION != 0) -static const char __section("bus_autoconf_format") __used usb_id_format[] = { - - /* Declare that three different sections use the same format */ - - "usb_host_id{256,:}" - "usb_device_id{256,:}" - "usb_dual_id{256,:}" - - /* List size of fields in the usb_device_id structure */ - - "mf_vendor{" U16_XOR ",1}" - "mf_product{" U16_XOR ",1}" - "mf_dev_lo{" U16_XOR ",1}" - "mf_dev_hi{" U16_XOR ",1}" - - "mf_dev_class{" U16_XOR ",1}" - "mf_dev_subclass{" U16_XOR ",1}" - "mf_dev_protocol{" U16_XOR ",1}" - "mf_int_class{" U16_XOR ",1}" - - "mf_int_subclass{" U16_XOR ",1}" - "mf_int_protocol{" U16_XOR ",1}" - "unused{" U16_XOR ",6}" - - "idVendor[0]{" U16_XOR ",8}" - "idVendor[1]{" U16_XOR ",8}" - "idProduct[0]{" U16_XOR ",8}" - "idProduct[1]{" U16_XOR ",8}" - "bcdDevice_lo[0]{" U16_XOR ",8}" - "bcdDevice_lo[1]{" U16_XOR ",8}" - "bcdDevice_hi[0]{" U16_XOR ",8}" - "bcdDevice_hi[1]{" U16_XOR ",8}" - - "bDeviceClass{0,8}" - "bDeviceSubClass{0,8}" - "bDeviceProtocol{0,8}" - "bInterfaceClass{0,8}" - "bInterfaceSubClass{0,8}" - "bInterfaceProtocol{0,8}" - -#if USB_HAVE_COMPAT_LINUX - "mfl_vendor{" U16_XOR ",1}" - "mfl_product{" U16_XOR ",1}" - "mfl_dev_lo{" U16_XOR ",1}" - "mfl_dev_hi{" U16_XOR ",1}" - - "mfl_dev_class{" U16_XOR ",1}" - "mfl_dev_subclass{" U16_XOR ",1}" - "mfl_dev_protocol{" U16_XOR ",1}" - "mfl_int_class{" U16_XOR ",1}" - - "mfl_int_subclass{" U16_XOR ",1}" - "mfl_int_protocol{" U16_XOR ",1}" - "unused{" U16_XOR ",6}" -#endif -}; -#endif diff --git a/freebsd/sys/dev/usb/usbdi.h b/freebsd/sys/dev/usb/usbdi.h index 2be8eef1..147b5d5e 100644 --- a/freebsd/sys/dev/usb/usbdi.h +++ b/freebsd/sys/dev/usb/usbdi.h @@ -336,19 +336,19 @@ struct usb_device_id { unsigned long driver_info; } __aligned(32); -#define USB_STD_PNP_INFO "M16:mask;U16:vendor;U16:product;L16:product;G16:product;" \ - "U8:devclass;U8:devsubclass;U8:devprotocol;" \ +#define USB_STD_PNP_INFO "M16:mask;U16:vendor;U16:product;L16:release;G16:release;" \ + "U8:devclass;U8:devsubclass;U8:devproto;" \ "U8:intclass;U8:intsubclass;U8:intprotocol;" #define USB_STD_PNP_HOST_INFO USB_STD_PNP_INFO "T:mode=host;" #define USB_STD_PNP_DEVICE_INFO USB_STD_PNP_INFO "T:mode=device;" #define USB_PNP_HOST_INFO(table) \ - MODULE_PNP_INFO(USB_STD_PNP_HOST_INFO, usb, table, table, sizeof(table[0]), \ + MODULE_PNP_INFO(USB_STD_PNP_HOST_INFO, uhub, table, table, sizeof(table[0]), \ sizeof(table) / sizeof(table[0])) #define USB_PNP_DEVICE_INFO(table) \ - MODULE_PNP_INFO(USB_STD_PNP_DEVICE_INFO, usb, table, table, sizeof(table[0]), \ + MODULE_PNP_INFO(USB_STD_PNP_DEVICE_INFO, uhub, table, table, sizeof(table[0]), \ sizeof(table) / sizeof(table[0])) #define USB_PNP_DUAL_INFO(table) \ - MODULE_PNP_INFO(USB_STD_PNP_INFO, usb, table, table, sizeof(table[0]), \ + MODULE_PNP_INFO(USB_STD_PNP_INFO, uhub, table, table, sizeof(table[0]), \ sizeof(table) / sizeof(table[0])) /* check that the size of the structure above is correct */ diff --git a/freebsd/sys/dev/usb/wlan/if_rum.c b/freebsd/sys/dev/usb/wlan/if_rum.c index dbb12335..5c826cac 100644 --- a/freebsd/sys/dev/usb/wlan/if_rum.c +++ b/freebsd/sys/dev/usb/wlan/if_rum.c @@ -29,6 +29,8 @@ __FBSDID("$FreeBSD$"); * http://www.ralinktech.com.tw/ */ +#include <rtems/bsd/local/opt_wlan.h> + #include <sys/param.h> #include <sys/sockio.h> #include <sys/sysctl.h> @@ -1421,37 +1423,25 @@ rum_sendprot(struct rum_softc *sc, const struct mbuf *m, struct ieee80211_node *ni, int prot, int rate) { struct ieee80211com *ic = ni->ni_ic; - const struct ieee80211_frame *wh; struct rum_tx_data *data; struct mbuf *mprot; - int protrate, pktlen, flags, isshort; - uint16_t dur; + int protrate, flags; RUM_LOCK_ASSERT(sc); - KASSERT(prot == IEEE80211_PROT_RTSCTS || prot == IEEE80211_PROT_CTSONLY, - ("protection %d", prot)); - wh = mtod(m, const struct ieee80211_frame *); - pktlen = m->m_pkthdr.len + IEEE80211_CRC_LEN; + mprot = ieee80211_alloc_prot(ni, m, rate, prot); + if (mprot == NULL) { + if_inc_counter(ni->ni_vap->iv_ifp, IFCOUNTER_OERRORS, 1); + device_printf(sc->sc_dev, + "could not allocate mbuf for protection mode %d\n", prot); + return (ENOBUFS); + } protrate = ieee80211_ctl_rate(ic->ic_rt, rate); - - isshort = (ic->ic_flags & IEEE80211_F_SHPREAMBLE) != 0; - dur = ieee80211_compute_duration(ic->ic_rt, pktlen, rate, isshort) - + ieee80211_ack_duration(ic->ic_rt, rate, isshort); flags = 0; - if (prot == IEEE80211_PROT_RTSCTS) { - /* NB: CTS is the same size as an ACK */ - dur += ieee80211_ack_duration(ic->ic_rt, rate, isshort); + if (prot == IEEE80211_PROT_RTSCTS) flags |= RT2573_TX_NEED_ACK; - mprot = ieee80211_alloc_rts(ic, wh->i_addr1, wh->i_addr2, dur); - } else { - mprot = ieee80211_alloc_cts(ic, ni->ni_vap->iv_myaddr, dur); - } - if (mprot == NULL) { - /* XXX stat + msg */ - return (ENOBUFS); - } + data = STAILQ_FIRST(&sc->tx_free); STAILQ_REMOVE_HEAD(&sc->tx_free, next); sc->tx_nfree--; diff --git a/freebsd/sys/dev/usb/wlan/if_run.c b/freebsd/sys/dev/usb/wlan/if_run.c index 71f53dd3..41f97ba4 100644 --- a/freebsd/sys/dev/usb/wlan/if_run.c +++ b/freebsd/sys/dev/usb/wlan/if_run.c @@ -27,6 +27,8 @@ __FBSDID("$FreeBSD$"); * http://www.ralinktech.com/ */ +#include <rtems/bsd/local/opt_wlan.h> + #include <sys/param.h> #include <sys/sockio.h> #include <sys/sysctl.h> @@ -3485,7 +3487,6 @@ run_tx_mgt(struct run_softc *sc, struct mbuf *m, struct ieee80211_node *ni) struct rt2860_txwi *txwi; uint16_t dur; uint8_t ridx = rn->mgt_ridx; - uint8_t type; uint8_t xflags = 0; uint8_t wflags = 0; @@ -3493,8 +3494,6 @@ run_tx_mgt(struct run_softc *sc, struct mbuf *m, struct ieee80211_node *ni) wh = mtod(m, struct ieee80211_frame *); - type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK; - /* tell hardware to add timestamp for probe responses */ if ((wh->i_fc[0] & (IEEE80211_FC0_TYPE_MASK | IEEE80211_FC0_SUBTYPE_MASK)) == @@ -3545,57 +3544,35 @@ run_sendprot(struct run_softc *sc, const struct mbuf *m, struct ieee80211_node *ni, int prot, int rate) { struct ieee80211com *ic = ni->ni_ic; - struct ieee80211_frame *wh; struct run_tx_data *data; struct rt2870_txd *txd; struct rt2860_txwi *txwi; struct mbuf *mprot; int ridx; int protrate; - int ackrate; - int pktlen; - int isshort; - uint16_t dur; - uint8_t type; uint8_t wflags = 0; uint8_t xflags = 0; RUN_LOCK_ASSERT(sc, MA_OWNED); - KASSERT(prot == IEEE80211_PROT_RTSCTS || prot == IEEE80211_PROT_CTSONLY, - ("protection %d", prot)); - - wh = mtod(m, struct ieee80211_frame *); - pktlen = m->m_pkthdr.len + IEEE80211_CRC_LEN; - type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK; - - protrate = ieee80211_ctl_rate(ic->ic_rt, rate); - ackrate = ieee80211_ack_rate(ic->ic_rt, rate); - - isshort = (ic->ic_flags & IEEE80211_F_SHPREAMBLE) != 0; - dur = ieee80211_compute_duration(ic->ic_rt, pktlen, rate, isshort) - + ieee80211_ack_duration(ic->ic_rt, rate, isshort); - wflags = RT2860_TX_FRAG; - /* check that there are free slots before allocating the mbuf */ if (sc->sc_epq[0].tx_nfree == 0) /* let caller free mbuf */ return (ENOBUFS); - if (prot == IEEE80211_PROT_RTSCTS) { - /* NB: CTS is the same size as an ACK */ - dur += ieee80211_ack_duration(ic->ic_rt, rate, isshort); - xflags |= RT2860_TX_ACK; - mprot = ieee80211_alloc_rts(ic, wh->i_addr1, wh->i_addr2, dur); - } else { - mprot = ieee80211_alloc_cts(ic, ni->ni_vap->iv_myaddr, dur); - } + mprot = ieee80211_alloc_prot(ni, m, rate, prot); if (mprot == NULL) { if_inc_counter(ni->ni_vap->iv_ifp, IFCOUNTER_OERRORS, 1); RUN_DPRINTF(sc, RUN_DEBUG_XMIT, "could not allocate mbuf\n"); return (ENOBUFS); } + protrate = ieee80211_ctl_rate(ic->ic_rt, rate); + wflags = RT2860_TX_FRAG; + xflags = 0; + if (prot == IEEE80211_PROT_RTSCTS) + xflags |= RT2860_TX_ACK; + data = STAILQ_FIRST(&sc->sc_epq[0].tx_fh); STAILQ_REMOVE_HEAD(&sc->sc_epq[0].tx_fh, next); sc->sc_epq[0].tx_nfree--; @@ -3633,11 +3610,9 @@ run_tx_param(struct run_softc *sc, struct mbuf *m, struct ieee80211_node *ni, const struct ieee80211_bpf_params *params) { struct ieee80211com *ic = ni->ni_ic; - struct ieee80211_frame *wh; struct run_tx_data *data; struct rt2870_txd *txd; struct rt2860_txwi *txwi; - uint8_t type; uint8_t ridx; uint8_t rate; uint8_t opflags = 0; @@ -3648,9 +3623,6 @@ run_tx_param(struct run_softc *sc, struct mbuf *m, struct ieee80211_node *ni, KASSERT(params != NULL, ("no raw xmit params")); - wh = mtod(m, struct ieee80211_frame *); - type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK; - rate = params->ibp_rate0; if (!ieee80211_isratevalid(ic->ic_rt, rate)) { /* let caller free mbuf */ diff --git a/freebsd/sys/dev/usb/wlan/if_uath.c b/freebsd/sys/dev/usb/wlan/if_uath.c index e001d9ed..2b97060e 100644 --- a/freebsd/sys/dev/usb/wlan/if_uath.c +++ b/freebsd/sys/dev/usb/wlan/if_uath.c @@ -69,6 +69,9 @@ __FBSDID("$FreeBSD$"); * for these devices does not work in this way and so does not work * with this driver. */ + +#include <rtems/bsd/local/opt_wlan.h> + #include <sys/param.h> #include <sys/sockio.h> #include <sys/sysctl.h> diff --git a/freebsd/sys/dev/usb/wlan/if_upgt.c b/freebsd/sys/dev/usb/wlan/if_upgt.c index 8f19f63b..e1923bab 100644 --- a/freebsd/sys/dev/usb/wlan/if_upgt.c +++ b/freebsd/sys/dev/usb/wlan/if_upgt.c @@ -19,6 +19,8 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include <rtems/bsd/local/opt_wlan.h> + #include <sys/param.h> #include <sys/systm.h> #include <sys/kernel.h> diff --git a/freebsd/sys/dev/usb/wlan/if_ural.c b/freebsd/sys/dev/usb/wlan/if_ural.c index 2f237d7d..4de0a9c5 100644 --- a/freebsd/sys/dev/usb/wlan/if_ural.c +++ b/freebsd/sys/dev/usb/wlan/if_ural.c @@ -30,6 +30,8 @@ __FBSDID("$FreeBSD$"); * http://www.ralinktech.com/ */ +#include <rtems/bsd/local/opt_wlan.h> + #include <sys/param.h> #include <sys/sockio.h> #include <sys/sysctl.h> @@ -1132,37 +1134,23 @@ ural_sendprot(struct ural_softc *sc, const struct mbuf *m, struct ieee80211_node *ni, int prot, int rate) { struct ieee80211com *ic = ni->ni_ic; - const struct ieee80211_frame *wh; struct ural_tx_data *data; struct mbuf *mprot; - int protrate, ackrate, pktlen, flags, isshort; - uint16_t dur; + int protrate, flags; - KASSERT(prot == IEEE80211_PROT_RTSCTS || prot == IEEE80211_PROT_CTSONLY, - ("protection %d", prot)); - - wh = mtod(m, const struct ieee80211_frame *); - pktlen = m->m_pkthdr.len + IEEE80211_CRC_LEN; + mprot = ieee80211_alloc_prot(ni, m, rate, prot); + if (mprot == NULL) { + if_inc_counter(ni->ni_vap->iv_ifp, IFCOUNTER_OERRORS, 1); + device_printf(sc->sc_dev, + "could not allocate mbuf for protection mode %d\n", prot); + return ENOBUFS; + } protrate = ieee80211_ctl_rate(ic->ic_rt, rate); - ackrate = ieee80211_ack_rate(ic->ic_rt, rate); - - isshort = (ic->ic_flags & IEEE80211_F_SHPREAMBLE) != 0; - dur = ieee80211_compute_duration(ic->ic_rt, pktlen, rate, isshort) - + ieee80211_ack_duration(ic->ic_rt, rate, isshort); flags = RAL_TX_RETRY(7); - if (prot == IEEE80211_PROT_RTSCTS) { - /* NB: CTS is the same size as an ACK */ - dur += ieee80211_ack_duration(ic->ic_rt, rate, isshort); + if (prot == IEEE80211_PROT_RTSCTS) flags |= RAL_TX_ACK; - mprot = ieee80211_alloc_rts(ic, wh->i_addr1, wh->i_addr2, dur); - } else { - mprot = ieee80211_alloc_cts(ic, ni->ni_vap->iv_myaddr, dur); - } - if (mprot == NULL) { - /* XXX stat + msg */ - return ENOBUFS; - } + data = STAILQ_FIRST(&sc->tx_free); STAILQ_REMOVE_HEAD(&sc->tx_free, next); sc->tx_nfree--; diff --git a/freebsd/sys/dev/usb/wlan/if_urtw.c b/freebsd/sys/dev/usb/wlan/if_urtw.c index d94cbee7..309375f0 100644 --- a/freebsd/sys/dev/usb/wlan/if_urtw.c +++ b/freebsd/sys/dev/usb/wlan/if_urtw.c @@ -18,6 +18,9 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); + +#include <rtems/bsd/local/opt_wlan.h> + #include <sys/param.h> #include <sys/sockio.h> #include <sys/sysctl.h> @@ -64,6 +67,12 @@ __FBSDID("$FreeBSD$"); #include <dev/usb/wlan/if_urtwreg.h> #include <dev/usb/wlan/if_urtwvar.h> +/* copy some rate indices from if_rtwn_ridx.h */ +#define URTW_RIDX_CCK5 2 +#define URTW_RIDX_CCK11 3 +#define URTW_RIDX_OFDM6 4 +#define URTW_RIDX_OFDM24 8 + static SYSCTL_NODE(_hw_usb, OID_AUTO, urtw, CTLFLAG_RW, 0, "USB Realtek 8187L"); #ifdef URTW_DEBUG int urtw_debug = 0; @@ -681,7 +690,6 @@ static void urtw_ledtask(void *, int); static void urtw_watchdog(void *); static void urtw_set_multi(void *); static int urtw_isbmode(uint16_t); -static uint16_t urtw_rate2rtl(uint32_t); static uint16_t urtw_rtl2rate(uint32_t); static usb_error_t urtw_set_rate(struct urtw_softc *); static usb_error_t urtw_update_msr(struct urtw_softc *); @@ -865,7 +873,7 @@ urtw_attach(device_t dev) sc->sc_rts_retry = URTW_DEFAULT_RTS_RETRY; sc->sc_tx_retry = URTW_DEFAULT_TX_RETRY; - sc->sc_currate = 3; + sc->sc_currate = URTW_RIDX_CCK11; sc->sc_preamble_mode = urtw_preamble_mode; ic->ic_softc = sc; @@ -1655,14 +1663,17 @@ urtw_tx_start(struct urtw_softc *sc, struct ieee80211_node *ni, struct mbuf *m0, sc->sc_xfer[URTW_8187B_BULK_TX_VO] }; struct usb_xfer *xfer; - int dur = 0, rtsdur = 0, rtsenable = 0, ctsenable = 0, rate, - pkttime = 0, txdur = 0, isshort = 0, xferlen; + int dur = 0, rtsdur = 0, rtsenable = 0, ctsenable = 0, rate, type, + pkttime = 0, txdur = 0, isshort = 0, xferlen, ismcast; uint16_t acktime, rtstime, ctstime; uint32_t flags; usb_error_t error; URTW_ASSERT_LOCKED(sc); + ismcast = IEEE80211_IS_MULTICAST(wh->i_addr1); + type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK; + /* * Software crypto. */ @@ -1691,13 +1702,13 @@ urtw_tx_start(struct urtw_softc *sc, struct ieee80211_node *ni, struct mbuf *m0, ieee80211_radiotap_tx(vap, m0); } - if ((wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) == IEEE80211_FC0_TYPE_MGT || - (wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) == IEEE80211_FC0_TYPE_CTL || + if (type == IEEE80211_FC0_TYPE_MGT || + type == IEEE80211_FC0_TYPE_CTL || (m0->m_flags & M_EAPOL) != 0) { rate = tp->mgmtrate; } else { /* for data frames */ - if (IEEE80211_IS_MULTICAST(wh->i_addr1)) + if (ismcast) rate = tp->mcastrate; else if (tp->ucastrate != IEEE80211_FIXED_RATE_NONE) rate = tp->ucastrate; @@ -1707,7 +1718,7 @@ urtw_tx_start(struct urtw_softc *sc, struct ieee80211_node *ni, struct mbuf *m0, sc->sc_stats.txrates[sc->sc_currate]++; - if (IEEE80211_IS_MULTICAST(wh->i_addr1)) + if (ismcast) txdur = pkttime = urtw_compute_txtime(m0->m_pkthdr.len + IEEE80211_CRC_LEN, rate, 0, 0); else { @@ -1765,14 +1776,12 @@ urtw_tx_start(struct urtw_softc *sc, struct ieee80211_node *ni, struct mbuf *m0, flags |= URTW_TX_FLAG_CTS; if (rtsenable) { flags |= URTW_TX_FLAG_RTS; - flags |= (urtw_rate2rtl(11) & 0xf) << - URTW_TX_FLAG_RTSRATE_SHIFT; + flags |= URTW_RIDX_CCK5 << URTW_TX_FLAG_RTSRATE_SHIFT; tx->rtsdur = rtsdur; } tx->flag = htole32(flags); tx->txdur = txdur; - if ((wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) == - IEEE80211_FC0_TYPE_MGT && + if (type == IEEE80211_FC0_TYPE_MGT && (wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK) == IEEE80211_FC0_SUBTYPE_PROBE_RESP) tx->retry = 1; @@ -1787,7 +1796,7 @@ urtw_tx_start(struct urtw_softc *sc, struct ieee80211_node *ni, struct mbuf *m0, flags |= URTW_TX_FLAG_RTS; tx->rtsdur = rtsdur; } - flags |= (urtw_rate2rtl(11) & 0xf) << URTW_TX_FLAG_RTSRATE_SHIFT; + flags |= URTW_RIDX_CCK5 << URTW_TX_FLAG_RTSRATE_SHIFT; tx->flag = htole32(flags); tx->retry = 3; /* CW minimum */ tx->retry |= 7 << 4; /* CW maximum */ @@ -1800,7 +1809,7 @@ urtw_tx_start(struct urtw_softc *sc, struct ieee80211_node *ni, struct mbuf *m0, data->m = m0; if (sc->sc_flags & URTW_RTL8187B) { - switch (wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) { + switch (type) { case IEEE80211_FC0_TYPE_CTL: case IEEE80211_FC0_TYPE_MGT: xfer = sc->sc_xfer[URTW_8187B_BULK_TX_EP12]; @@ -1907,9 +1916,9 @@ urtw_set_rate(struct urtw_softc *sc) uint16_t data; usb_error_t error; - basic_rate = urtw_rate2rtl(48); - min_rr_rate = urtw_rate2rtl(12); - max_rr_rate = urtw_rate2rtl(48); + basic_rate = URTW_RIDX_OFDM24; + min_rr_rate = URTW_RIDX_OFDM6; + max_rr_rate = URTW_RIDX_OFDM24; urtw_write8_m(sc, URTW_RESP_RATE, max_rr_rate << URTW_RESP_MAX_RATE_SHIFT | @@ -1927,19 +1936,6 @@ fail: } static uint16_t -urtw_rate2rtl(uint32_t rate) -{ - unsigned int i; - - for (i = 0; i < nitems(urtw_ratetable); i++) { - if (rate == urtw_ratetable[i].reg) - return urtw_ratetable[i].val; - } - - return (3); -} - -static uint16_t urtw_rtl2rate(uint32_t rate) { unsigned int i; diff --git a/freebsd/sys/dev/usb/wlan/if_zyd.c b/freebsd/sys/dev/usb/wlan/if_zyd.c index e4243d8f..dee03bdd 100644 --- a/freebsd/sys/dev/usb/wlan/if_zyd.c +++ b/freebsd/sys/dev/usb/wlan/if_zyd.c @@ -28,6 +28,8 @@ __FBSDID("$FreeBSD$"); * ZyDAS ZD1211/ZD1211B USB WLAN driver. */ +#include <rtems/bsd/local/opt_wlan.h> + #include <sys/param.h> #include <sys/sockio.h> #include <sys/sysctl.h> @@ -2443,7 +2445,7 @@ zyd_tx_start(struct zyd_softc *sc, struct mbuf *m0, struct ieee80211_node *ni) struct ieee80211_frame *wh; const struct ieee80211_txparam *tp = ni->ni_txparms; struct ieee80211_key *k; - int rate, totlen; + int rate, totlen, type, ismcast; static const uint8_t ratediv[] = ZYD_TX_RATEDIV; uint8_t phy; uint16_t pktlen; @@ -2454,13 +2456,16 @@ zyd_tx_start(struct zyd_softc *sc, struct mbuf *m0, struct ieee80211_node *ni) STAILQ_REMOVE_HEAD(&sc->tx_free, next); sc->tx_nfree--; - if ((wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) == IEEE80211_FC0_TYPE_MGT || - (wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) == IEEE80211_FC0_TYPE_CTL || + ismcast = IEEE80211_IS_MULTICAST(wh->i_addr1); + type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK; + + if (type == IEEE80211_FC0_TYPE_MGT || + type == IEEE80211_FC0_TYPE_CTL || (m0->m_flags & M_EAPOL) != 0) { rate = tp->mgmtrate; } else { /* for data frames */ - if (IEEE80211_IS_MULTICAST(wh->i_addr1)) + if (ismcast) rate = tp->mcastrate; else if (tp->ucastrate != IEEE80211_FIXED_RATE_NONE) rate = tp->ucastrate; @@ -2498,7 +2503,7 @@ zyd_tx_start(struct zyd_softc *sc, struct mbuf *m0, struct ieee80211_node *ni) desc->len = htole16(totlen); desc->flags = ZYD_TX_FLAG_BACKOFF; - if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) { + if (!ismcast) { /* multicast frames are not sent at OFDM rates in 802.11b/g */ if (totlen > vap->iv_rtsthreshold) { desc->flags |= ZYD_TX_FLAG_RTS; diff --git a/freebsd/sys/i386/include/machine/intr_machdep.h b/freebsd/sys/i386/include/machine/intr_machdep.h index 06272209..1ac8f489 100644 --- a/freebsd/sys/i386/include/machine/intr_machdep.h +++ b/freebsd/sys/i386/include/machine/intr_machdep.h @@ -132,6 +132,7 @@ struct intsrc { u_long *is_straycount; u_int is_index; u_int is_handlers; + u_int is_domain; u_int is_cpu; }; @@ -158,7 +159,8 @@ void elcr_write_trigger(u_int irq, enum intr_trigger trigger); void intr_add_cpu(u_int cpu); #endif int intr_add_handler(const char *name, int vector, driver_filter_t filter, - driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep); + driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep, + int domain); #ifdef SMP int intr_bind(u_int vector, u_char cpu); #endif @@ -166,7 +168,7 @@ int intr_config_intr(int vector, enum intr_trigger trig, enum intr_polarity pol); int intr_describe(u_int vector, void *ih, const char *descr); void intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame); -u_int intr_next_cpu(void); +u_int intr_next_cpu(int domain); struct intsrc *intr_lookup_source(int vector); int intr_register_pic(struct pic *pic); int intr_register_source(struct intsrc *isrc); diff --git a/freebsd/sys/kern/init_main.c b/freebsd/sys/kern/init_main.c index ea827ebe..86bc11fc 100644 --- a/freebsd/sys/kern/init_main.c +++ b/freebsd/sys/kern/init_main.c @@ -89,6 +89,7 @@ __FBSDID("$FreeBSD$"); #include <vm/vm.h> #include <vm/vm_param.h> +#include <vm/vm_extern.h> #include <vm/pmap.h> #include <vm/vm_map.h> #include <sys/copyright.h> @@ -579,7 +580,7 @@ proc0_init(void *dummy __unused) p->p_limit->pl_rlimit[RLIMIT_STACK].rlim_cur = dflssiz; p->p_limit->pl_rlimit[RLIMIT_STACK].rlim_max = maxssiz; /* Cast to avoid overflow on i386/PAE. */ - pageablemem = ptoa((vm_paddr_t)vm_cnt.v_free_count); + pageablemem = ptoa((vm_paddr_t)vm_free_count()); p->p_limit->pl_rlimit[RLIMIT_RSS].rlim_cur = p->p_limit->pl_rlimit[RLIMIT_RSS].rlim_max = pageablemem; p->p_limit->pl_rlimit[RLIMIT_MEMLOCK].rlim_cur = pageablemem / 3; @@ -726,10 +727,6 @@ start_init(void *dummy) struct thread *td; struct proc *p; - mtx_lock(&Giant); - - GIANT_REQUIRED; - TSENTER(); /* Here so we don't overlap with mi_startup. */ td = curthread; @@ -824,7 +821,6 @@ start_init(void *dummy) * to user mode as init! */ if ((error = sys_execve(td, &args)) == EJUSTRETURN) { - mtx_unlock(&Giant); TSEXIT(); return; } diff --git a/freebsd/sys/kern/kern_mib.c b/freebsd/sys/kern/kern_mib.c index 4f520ef3..3fd48334 100644 --- a/freebsd/sys/kern/kern_mib.c +++ b/freebsd/sys/kern/kern_mib.c @@ -215,7 +215,7 @@ sysctl_hw_usermem(SYSCTL_HANDLER_ARGS) { u_long val; - val = ctob(physmem - vm_cnt.v_wire_count); + val = ctob(physmem - vm_wire_count()); return (sysctl_handle_long(oidp, &val, 0, req)); } diff --git a/freebsd/sys/kern/kern_synch.c b/freebsd/sys/kern/kern_synch.c index f6485a86..59f00ed4 100644 --- a/freebsd/sys/kern/kern_synch.c +++ b/freebsd/sys/kern/kern_synch.c @@ -141,7 +141,11 @@ _sleep(void *ident, struct lock_object *lock, int priority, struct thread *td; struct lock_class *class; uintptr_t lock_state; +#ifndef __rtems__ int catch, pri, rval, sleepq_flags; +#else /* __rtems__ */ + int pri, rval, sleepq_flags; +#endif /* __rtems__ */ WITNESS_SAVE_DECL(lock_witness); td = curthread; @@ -174,7 +178,6 @@ _sleep(void *ident, struct lock_object *lock, int priority, catch = priority & PCATCH; pri = priority & PRIMASK; #else /* __rtems__ */ - (void)catch; pri = priority; #endif /* __rtems__ */ @@ -322,16 +325,16 @@ msleep_spin_sbt(void *ident, struct mtx *mtx, const char *wmesg, #endif /* __rtems__ */ /* - * pause() delays the calling thread by the given number of system ticks. - * During cold bootup, pause() uses the DELAY() function instead of - * the tsleep() function to do the waiting. The "timo" argument must be - * greater than or equal to zero. A "timo" value of zero is equivalent - * to a "timo" value of one. + * pause_sbt() delays the calling thread by the given signed binary + * time. During cold bootup, pause_sbt() uses the DELAY() function + * instead of the _sleep() function to do the waiting. The "sbt" + * argument must be greater than or equal to zero. A "sbt" value of + * zero is equivalent to a "sbt" value of one tick. */ int pause_sbt(const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags) { - KASSERT(sbt >= 0, ("pause: timeout must be >= 0")); + KASSERT(sbt >= 0, ("pause_sbt: timeout must be >= 0")); /* silently convert invalid timeouts */ if (sbt == 0) @@ -352,10 +355,13 @@ pause_sbt(const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags) sbt = howmany(sbt, SBT_1US); if (sbt > 0) DELAY(sbt); - return (0); + return (EWOULDBLOCK); } -#endif /* __rtems__ */ + return (_sleep(&pause_wchan[curcpu], NULL, + (flags & C_CATCH) ? PCATCH : 0, wmesg, sbt, pr, flags)); +#else /* __rtems__ */ return (_sleep(&pause_wchan[curcpu], NULL, 0, wmesg, sbt, pr, flags)); +#endif /* __rtems__ */ } /* diff --git a/freebsd/sys/kern/kern_sysctl.c b/freebsd/sys/kern/kern_sysctl.c index 02078a32..8b27a5c6 100644 --- a/freebsd/sys/kern/kern_sysctl.c +++ b/freebsd/sys/kern/kern_sysctl.c @@ -193,7 +193,7 @@ sysctl_load_tunable_by_oid_locked(struct sysctl_oid *oidp) struct sysctl_req req; struct sysctl_oid *curr; char *penv = NULL; - char path[64]; + char path[96]; ssize_t rem = sizeof(path); ssize_t len; uint8_t val_8; diff --git a/freebsd/sys/kern/kern_time.c b/freebsd/sys/kern/kern_time.c index 61dc7c73..a9c0547a 100644 --- a/freebsd/sys/kern/kern_time.c +++ b/freebsd/sys/kern/kern_time.c @@ -576,7 +576,8 @@ kern_clock_nanosleep(struct thread *td, clockid_t clock_id, int flags, } while (error == 0 && is_abs_real && td->td_rtcgen == 0); td->td_rtcgen = 0; if (error != EWOULDBLOCK) { - TIMESEL(&sbtt, tmp); + if (TIMESEL(&sbtt, tmp)) + sbtt += tc_tick_sbt; if (sbtt >= sbt) return (0); if (error == ERESTART) diff --git a/freebsd/sys/kern/subr_eventhandler.c b/freebsd/sys/kern/subr_eventhandler.c index 92cd9726..e07248bf 100644 --- a/freebsd/sys/kern/subr_eventhandler.c +++ b/freebsd/sys/kern/subr_eventhandler.c @@ -92,9 +92,10 @@ eventhandler_find_or_create_list(const char *name) CTR2(KTR_EVH, "%s: creating list \"%s\"", __func__, name); list = new_list; TAILQ_INIT(&list->el_entries); - mtx_init(&list->el_lock, name, "eventhandler list", MTX_DEF); list->el_name = (char *)(list + 1); strcpy(list->el_name, name); + mtx_init(&list->el_lock, list->el_name, "eventhandler list", + MTX_DEF); TAILQ_INSERT_HEAD(&eventhandler_lists, list, el_link); } } diff --git a/freebsd/sys/kern/subr_pcpu.c b/freebsd/sys/kern/subr_pcpu.c index 1d230b68..67a1f528 100644 --- a/freebsd/sys/kern/subr_pcpu.c +++ b/freebsd/sys/kern/subr_pcpu.c @@ -155,7 +155,7 @@ pcpu_zones_startup(void) pcpu_zone_ptr = uma_zcreate("ptr pcpu", sizeof(void *), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_PCPU); } -SYSINIT(pcpu_zones, SI_SUB_KMEM, SI_ORDER_ANY, pcpu_zones_startup, NULL); +SYSINIT(pcpu_zones, SI_SUB_VM, SI_ORDER_ANY, pcpu_zones_startup, NULL); #ifndef __rtems__ /* diff --git a/freebsd/sys/kern/subr_sleepqueue.c b/freebsd/sys/kern/subr_sleepqueue.c index 6ecefc70..b9de1580 100644 --- a/freebsd/sys/kern/subr_sleepqueue.c +++ b/freebsd/sys/kern/subr_sleepqueue.c @@ -1403,7 +1403,7 @@ void sleepq_chains_remove_matching(bool (*matches)(struct thread *)) { struct sleepqueue_chain *sc; - struct sleepqueue *sq; + struct sleepqueue *sq, *sq1; int i, wakeup_swapper; wakeup_swapper = 0; @@ -1412,7 +1412,7 @@ sleepq_chains_remove_matching(bool (*matches)(struct thread *)) continue; } mtx_lock_spin(&sc->sc_lock); - LIST_FOREACH(sq, &sc->sc_queues, sq_hash) { + LIST_FOREACH_SAFE(sq, &sc->sc_queues, sq_hash, sq1) { for (i = 0; i < NR_SLEEPQS; ++i) { wakeup_swapper |= sleepq_remove_matching(sq, i, matches, 0); diff --git a/freebsd/sys/kern/subr_uio.c b/freebsd/sys/kern/subr_uio.c index 4979bcf1..58db0ffc 100644 --- a/freebsd/sys/kern/subr_uio.c +++ b/freebsd/sys/kern/subr_uio.c @@ -524,8 +524,8 @@ copyout_unmap(struct thread *td, vm_offset_t addr, size_t sz) /* * XXXKIB The temporal implementation of fue*() functions which do not * handle usermode -1 properly, mixing it with the fault code. Keep - * this until MD code is written. Currently sparc64 and mips do not - * have proper implementation. + * this until MD code is written. Currently sparc64 does not have a + * proper implementation. */ int diff --git a/freebsd/sys/kern/sys_generic.c b/freebsd/sys/kern/sys_generic.c index 1c46bd29..0d5193c7 100644 --- a/freebsd/sys/kern/sys_generic.c +++ b/freebsd/sys/kern/sys_generic.c @@ -218,7 +218,7 @@ sys_read(td, uap) auio.uio_resid = uap->nbyte; auio.uio_segflg = UIO_USERSPACE; error = kern_readv(td, uap->fd, &auio); - return(error); + return (error); } /* @@ -381,7 +381,7 @@ dofileread(td, fd, fp, auio, offset, flags) /* Finish zero length reads right here */ if (auio->uio_resid == 0) { td->td_retval[0] = 0; - return(0); + return (0); } auio->uio_rw = UIO_READ; auio->uio_offset = offset; @@ -432,7 +432,7 @@ sys_write(td, uap) auio.uio_resid = uap->nbyte; auio.uio_segflg = UIO_USERSPACE; error = kern_writev(td, uap->fd, &auio); - return(error); + return (error); } /* @@ -471,7 +471,7 @@ kern_pwrite(struct thread *td, int fd, const void *buf, size_t nbyte, auio.uio_resid = nbyte; auio.uio_segflg = UIO_USERSPACE; error = kern_pwritev(td, fd, &auio, offset); - return(error); + return (error); } #if defined(COMPAT_FREEBSD6) diff --git a/freebsd/sys/kern/uipc_syscalls.c b/freebsd/sys/kern/uipc_syscalls.c index f2aba363..6f3c95c0 100644 --- a/freebsd/sys/kern/uipc_syscalls.c +++ b/freebsd/sys/kern/uipc_syscalls.c @@ -60,6 +60,7 @@ __FBSDID("$FreeBSD$"); #include <sys/socket.h> #include <sys/socketvar.h> #include <sys/syscallsubr.h> +#include <sys/uio.h> #ifdef KTRACE #include <sys/ktrace.h> #endif diff --git a/freebsd/sys/net/bpf.c b/freebsd/sys/net/bpf.c index e6ad9e25..24927e8b 100644 --- a/freebsd/sys/net/bpf.c +++ b/freebsd/sys/net/bpf.c @@ -292,7 +292,7 @@ bpf_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset, void *src, #ifndef __rtems__ case BPF_BUFMODE_ZBUF: - d->bd_zcopy++; + counter_u64_add(d->bd_zcopy, 1); return (bpf_zerocopy_append_bytes(d, buf, offset, src, len)); #endif /* __rtems__ */ @@ -314,7 +314,7 @@ bpf_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset, void *src, #ifndef __rtems__ case BPF_BUFMODE_ZBUF: - d->bd_zcopy++; + counter_u64_add(d->bd_zcopy, 1); return (bpf_zerocopy_append_mbuf(d, buf, offset, src, len)); #endif /* __rtems__ */ @@ -936,6 +936,15 @@ bpfopen(struct cdev *dev, int flags, int fmt, struct thread *td) } #endif /* __rtems__ */ + /* Setup counters */ + d->bd_rcount = counter_u64_alloc(M_WAITOK); + d->bd_dcount = counter_u64_alloc(M_WAITOK); + d->bd_fcount = counter_u64_alloc(M_WAITOK); + d->bd_wcount = counter_u64_alloc(M_WAITOK); + d->bd_wfcount = counter_u64_alloc(M_WAITOK); + d->bd_wdcount = counter_u64_alloc(M_WAITOK); + d->bd_zcopy = counter_u64_alloc(M_WAITOK); + /* * For historical reasons, perform a one-time initialization call to * the buffer routines, even though we're not yet committed to a @@ -1179,22 +1188,22 @@ bpfwrite(struct bpf_d *d, struct uio *uio, int ioflag) return (error); BPF_PID_REFRESH_CUR(d); - d->bd_wcount++; + counter_u64_add(d->bd_wcount, 1); /* XXX: locking required */ if (d->bd_bif == NULL) { - d->bd_wdcount++; + counter_u64_add(d->bd_wdcount, 1); return (ENXIO); } ifp = d->bd_bif->bif_ifp; if ((ifp->if_flags & IFF_UP) == 0) { - d->bd_wdcount++; + counter_u64_add(d->bd_wdcount, 1); return (ENETDOWN); } if (uio->uio_resid == 0) { - d->bd_wdcount++; + counter_u64_add(d->bd_wdcount, 1); return (0); } @@ -1205,10 +1214,10 @@ bpfwrite(struct bpf_d *d, struct uio *uio, int ioflag) error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp, &m, &dst, &hlen, d); if (error) { - d->bd_wdcount++; + counter_u64_add(d->bd_wdcount, 1); return (error); } - d->bd_wfcount++; + counter_u64_add(d->bd_wfcount, 1); if (d->bd_hdrcmplt) dst.sa_family = pseudo_AF_HDRCMPLT; @@ -1244,7 +1253,7 @@ bpfwrite(struct bpf_d *d, struct uio *uio, int ioflag) error = (*ifp->if_output)(ifp, m, &dst, &ro); if (error) - d->bd_wdcount++; + counter_u64_add(d->bd_wdcount, 1); if (mc != NULL) { if (error == 0) @@ -1283,13 +1292,13 @@ reset_d(struct bpf_d *d) } if (bpf_canwritebuf(d)) d->bd_slen = 0; - d->bd_rcount = 0; - d->bd_dcount = 0; - d->bd_fcount = 0; - d->bd_wcount = 0; - d->bd_wfcount = 0; - d->bd_wdcount = 0; - d->bd_zcopy = 0; + counter_u64_zero(d->bd_rcount); + counter_u64_zero(d->bd_dcount); + counter_u64_zero(d->bd_fcount); + counter_u64_zero(d->bd_wcount); + counter_u64_zero(d->bd_wfcount); + counter_u64_zero(d->bd_wdcount); + counter_u64_zero(d->bd_zcopy); } /* @@ -1667,8 +1676,8 @@ bpfioctl(struct bpf_d *d, u_long cmd, caddr_t addr, int flags, struct bpf_stat *bs = (struct bpf_stat *)addr; /* XXXCSJP overflow */ - bs->bs_recv = d->bd_rcount; - bs->bs_drop = d->bd_dcount; + bs->bs_recv = (u_int)counter_u64_fetch(d->bd_rcount); + bs->bs_drop = (u_int)counter_u64_fetch(d->bd_dcount); break; } @@ -2242,8 +2251,7 @@ bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) * write lock, too */ - /* XXX: Do not protect counter for the sake of performance. */ - ++d->bd_rcount; + counter_u64_add(d->bd_rcount, 1); /* * NB: We dont call BPF_CHECK_DIRECTION() here since there is no * way for the caller to indiciate to us whether this packet @@ -2263,7 +2271,7 @@ bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) */ BPFD_LOCK(d); - d->bd_fcount++; + counter_u64_add(d->bd_fcount, 1); if (gottime < bpf_ts_quality(d->bd_tstamp)) gottime = bpf_gettime(&bt, d->bd_tstamp, NULL); #ifdef MAC @@ -2310,7 +2318,7 @@ bpf_mtap(struct bpf_if *bp, struct mbuf *m) LIST_FOREACH(d, &bp->bif_dlist, bd_next) { if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp)) continue; - ++d->bd_rcount; + counter_u64_add(d->bd_rcount, 1); #ifdef BPF_JITTER bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL; /* XXX We cannot handle multiple mbufs. */ @@ -2322,7 +2330,7 @@ bpf_mtap(struct bpf_if *bp, struct mbuf *m) if (slen != 0) { BPFD_LOCK(d); - d->bd_fcount++; + counter_u64_add(d->bd_fcount, 1); if (gottime < bpf_ts_quality(d->bd_tstamp)) gottime = bpf_gettime(&bt, d->bd_tstamp, m); #ifdef MAC @@ -2373,12 +2381,12 @@ bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m) LIST_FOREACH(d, &bp->bif_dlist, bd_next) { if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp)) continue; - ++d->bd_rcount; + counter_u64_add(d->bd_rcount, 1); slen = bpf_filter(d->bd_rfilter, (u_char *)&mb, pktlen, 0); if (slen != 0) { BPFD_LOCK(d); - d->bd_fcount++; + counter_u64_add(d->bd_fcount, 1); if (gottime < bpf_ts_quality(d->bd_tstamp)) gottime = bpf_gettime(&bt, d->bd_tstamp, m); #ifdef MAC @@ -2530,7 +2538,7 @@ catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, * buffer model. */ bpf_buffull(d); - ++d->bd_dcount; + counter_u64_add(d->bd_dcount, 1); return; } KASSERT(!d->bd_hbuf_in_use, ("hold buffer is in use")); @@ -2630,6 +2638,15 @@ bpf_freed(struct bpf_d *d) if (d->bd_wfilter != NULL) free((caddr_t)d->bd_wfilter, M_BPF); mtx_destroy(&d->bd_lock); + + counter_u64_free(d->bd_rcount); + counter_u64_free(d->bd_dcount); + counter_u64_free(d->bd_fcount); + counter_u64_free(d->bd_wcount); + counter_u64_free(d->bd_wfcount); + counter_u64_free(d->bd_wdcount); + counter_u64_free(d->bd_zcopy); + } /* @@ -3110,12 +3127,12 @@ bpf_zero_counters(void) BPFIF_RLOCK(bp); LIST_FOREACH(bd, &bp->bif_dlist, bd_next) { BPFD_LOCK(bd); - bd->bd_rcount = 0; - bd->bd_dcount = 0; - bd->bd_fcount = 0; - bd->bd_wcount = 0; - bd->bd_wfcount = 0; - bd->bd_zcopy = 0; + counter_u64_zero(bd->bd_rcount); + counter_u64_zero(bd->bd_dcount); + counter_u64_zero(bd->bd_fcount); + counter_u64_zero(bd->bd_wcount); + counter_u64_zero(bd->bd_wfcount); + counter_u64_zero(bd->bd_zcopy); BPFD_UNLOCK(bd); } BPFIF_RUNLOCK(bp); @@ -3142,9 +3159,9 @@ bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd) #ifndef __rtems__ d->bd_async = bd->bd_async; #endif /* __rtems__ */ - d->bd_rcount = bd->bd_rcount; - d->bd_dcount = bd->bd_dcount; - d->bd_fcount = bd->bd_fcount; + d->bd_rcount = counter_u64_fetch(bd->bd_rcount); + d->bd_dcount = counter_u64_fetch(bd->bd_dcount); + d->bd_fcount = counter_u64_fetch(bd->bd_fcount); d->bd_sig = bd->bd_sig; d->bd_slen = bd->bd_slen; d->bd_hlen = bd->bd_hlen; @@ -3153,10 +3170,10 @@ bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd) strlcpy(d->bd_ifname, bd->bd_bif->bif_ifp->if_xname, IFNAMSIZ); d->bd_locked = bd->bd_locked; - d->bd_wcount = bd->bd_wcount; - d->bd_wdcount = bd->bd_wdcount; - d->bd_wfcount = bd->bd_wfcount; - d->bd_zcopy = bd->bd_zcopy; + d->bd_wcount = counter_u64_fetch(bd->bd_wcount); + d->bd_wdcount = counter_u64_fetch(bd->bd_wdcount); + d->bd_wfcount = counter_u64_fetch(bd->bd_wfcount); + d->bd_zcopy = counter_u64_fetch(bd->bd_zcopy); d->bd_bufmode = bd->bd_bufmode; } diff --git a/freebsd/sys/net/bpfdesc.h b/freebsd/sys/net/bpfdesc.h index 3d8e69ce..695d3d40 100644 --- a/freebsd/sys/net/bpfdesc.h +++ b/freebsd/sys/net/bpfdesc.h @@ -45,6 +45,7 @@ #include <sys/selinfo.h> #include <sys/queue.h> #include <sys/conf.h> +#include <sys/counter.h> #include <net/if.h> /* @@ -76,8 +77,8 @@ struct bpf_d { struct bpf_insn *bd_rfilter; /* read filter code */ struct bpf_insn *bd_wfilter; /* write filter code */ void *bd_bfilter; /* binary filter code */ - u_int64_t bd_rcount; /* number of packets received */ - u_int64_t bd_dcount; /* number of packets dropped */ + counter_u64_t bd_rcount; /* number of packets received */ + counter_u64_t bd_dcount; /* number of packets dropped */ u_char bd_promisc; /* true if listening promiscuously */ u_char bd_state; /* idle, waiting, or timed out */ @@ -96,14 +97,14 @@ struct bpf_d { struct mtx bd_lock; /* per-descriptor lock */ struct callout bd_callout; /* for BPF timeouts with select */ struct label *bd_label; /* MAC label for descriptor */ - u_int64_t bd_fcount; /* number of packets which matched filter */ + counter_u64_t bd_fcount; /* number of packets which matched filter */ pid_t bd_pid; /* PID which created descriptor */ int bd_locked; /* true if descriptor is locked */ u_int bd_bufmode; /* Current buffer mode. */ - u_int64_t bd_wcount; /* number of packets written */ - u_int64_t bd_wfcount; /* number of packets that matched write filter */ - u_int64_t bd_wdcount; /* number of packets dropped during a write */ - u_int64_t bd_zcopy; /* number of zero copy operations */ + counter_u64_t bd_wcount; /* number of packets written */ + counter_u64_t bd_wfcount; /* number of packets that matched write filter */ + counter_u64_t bd_wdcount; /* number of packets dropped during a write */ + counter_u64_t bd_zcopy; /* number of zero copy operations */ u_char bd_compat32; /* 32-bit stream on LP64 system */ }; diff --git a/freebsd/sys/net/ethernet.h b/freebsd/sys/net/ethernet.h index 5cd1dc50..fa75c1df 100644 --- a/freebsd/sys/net/ethernet.h +++ b/freebsd/sys/net/ethernet.h @@ -344,6 +344,7 @@ struct ether_vlan_header { #define ETHERTYPE_PPPOE 0x8864 /* PPP Over Ethernet Session Stage */ #define ETHERTYPE_LANPROBE 0x8888 /* HP LanProbe test? */ #define ETHERTYPE_PAE 0x888e /* EAPOL PAE/802.1x */ +#define ETHERTYPE_QINQ 0x88A8 /* 802.1ad VLAN stacking */ #define ETHERTYPE_LOOPBACK 0x9000 /* Loopback: used to test interfaces */ #define ETHERTYPE_LBACK ETHERTYPE_LOOPBACK /* DEC MOP loopback */ #define ETHERTYPE_XNSSM 0x9001 /* 3Com (Formerly Bridge Communications), XNS Systems Management */ @@ -384,6 +385,20 @@ struct ether_vlan_header { } \ } while (0) +/* + * Names for 802.1q priorities ("802.1p"). Notice that in this scheme, + * (0 < 1), allowing default 0-tagged traffic to take priority over background + * tagged traffic. + */ +#define IEEE8021Q_PCP_BK 1 /* Background (lowest) */ +#define IEEE8021Q_PCP_BE 0 /* Best effort (default) */ +#define IEEE8021Q_PCP_EE 2 /* Excellent effort */ +#define IEEE8021Q_PCP_CA 3 /* Critical applications */ +#define IEEE8021Q_PCP_VI 4 /* Video, < 100ms latency */ +#define IEEE8021Q_PCP_VO 5 /* Video, < 10ms latency */ +#define IEEE8021Q_PCP_IC 6 /* Internetwork control */ +#define IEEE8021Q_PCP_NC 7 /* Network control (highest) */ + #ifdef _KERNEL struct ifnet; @@ -405,6 +420,8 @@ extern char *ether_sprintf(const u_int8_t *); void ether_vlan_mtap(struct bpf_if *, struct mbuf *, void *, u_int); struct mbuf *ether_vlanencap(struct mbuf *, uint16_t); +bool ether_8021q_frame(struct mbuf **mp, struct ifnet *ife, struct ifnet *p, + uint16_t vid, uint8_t pcp); #ifdef _SYS_EVENTHANDLER_H_ /* new ethernet interface attached event */ diff --git a/freebsd/sys/net/if.c b/freebsd/sys/net/if.c index 67252bd4..bbf2ddd0 100644 --- a/freebsd/sys/net/if.c +++ b/freebsd/sys/net/if.c @@ -59,6 +59,7 @@ #include <sys/sockio.h> #include <sys/syslog.h> #include <sys/sysctl.h> +#include <sys/sysent.h> #include <sys/taskqueue.h> #include <sys/domain.h> #include <sys/jail.h> @@ -98,10 +99,59 @@ #include <security/mac/mac_framework.h> +/* + * Consumers of struct ifreq such as tcpdump assume no pad between ifr_name + * and ifr_ifru when it is used in SIOCGIFCONF. + */ +_Static_assert(sizeof(((struct ifreq *)0)->ifr_name) == + offsetof(struct ifreq, ifr_ifru), "gap between ifr_name and ifr_ifru"); + #ifdef COMPAT_FREEBSD32 #include <sys/mount.h> #include <compat/freebsd32/freebsd32.h> + +struct ifreq_buffer32 { + uint32_t length; /* (size_t) */ + uint32_t buffer; /* (void *) */ +}; + +/* + * Interface request structure used for socket + * ioctl's. All interface ioctl's must have parameter + * definitions which begin with ifr_name. The + * remainder may be interface specific. + */ +struct ifreq32 { + char ifr_name[IFNAMSIZ]; /* if name, e.g. "en0" */ + union { + struct sockaddr ifru_addr; + struct sockaddr ifru_dstaddr; + struct sockaddr ifru_broadaddr; + struct ifreq_buffer32 ifru_buffer; + short ifru_flags[2]; + short ifru_index; + int ifru_jid; + int ifru_metric; + int ifru_mtu; + int ifru_phys; + int ifru_media; + uint32_t ifru_data; + int ifru_cap[2]; + u_int ifru_fib; + u_char ifru_vlan_pcp; + } ifr_ifru; +}; +CTASSERT(sizeof(struct ifreq) == sizeof(struct ifreq32)); +CTASSERT(__offsetof(struct ifreq, ifr_ifru) == + __offsetof(struct ifreq32, ifr_ifru)); +#endif + +union ifreq_union { + struct ifreq ifr; +#ifdef COMPAT_FREEBSD32 + struct ifreq32 ifr32; #endif +}; SYSCTL_NODE(_net, PF_LINK, link, CTLFLAG_RW, 0, "Link layers"); SYSCTL_NODE(_net_link, 0, generic, CTLFLAG_RW, 0, "Generic link-management"); @@ -487,6 +537,7 @@ if_alloc(u_char type) for (int i = 0; i < IFCOUNTERS; i++) ifp->if_counters[i] = counter_u64_alloc(M_WAITOK); ifp->if_get_counter = if_get_counter_default; + ifp->if_pcp = IFNET_PCP_NONE; ifnet_setbyindex(ifp->if_index, ifp); return (ifp); } @@ -2306,6 +2357,75 @@ ifunit(const char *name) return (ifp); } +static void * +ifr_buffer_get_buffer(struct thread *td, void *data) +{ + union ifreq_union *ifrup; + + ifrup = data; +#ifdef COMPAT_FREEBSD32 + if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) + return ((void *)(uintptr_t) + ifrup->ifr32.ifr_ifru.ifru_buffer.buffer); +#endif + return (ifrup->ifr.ifr_ifru.ifru_buffer.buffer); +} + +static void +ifr_buffer_set_buffer_null(struct thread *td, void *data) +{ + union ifreq_union *ifrup; + + ifrup = data; +#ifdef COMPAT_FREEBSD32 + if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) + ifrup->ifr32.ifr_ifru.ifru_buffer.buffer = 0; + else +#endif + ifrup->ifr.ifr_ifru.ifru_buffer.buffer = NULL; +} + +static size_t +ifr_buffer_get_length(struct thread *td, void *data) +{ + union ifreq_union *ifrup; + + ifrup = data; +#ifdef COMPAT_FREEBSD32 + if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) + return (ifrup->ifr32.ifr_ifru.ifru_buffer.length); +#endif + return (ifrup->ifr.ifr_ifru.ifru_buffer.length); +} + +static void +ifr_buffer_set_length(struct thread *td, void *data, size_t len) +{ + union ifreq_union *ifrup; + + ifrup = data; +#ifdef COMPAT_FREEBSD32 + if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) + ifrup->ifr32.ifr_ifru.ifru_buffer.length = len; + else +#endif + ifrup->ifr.ifr_ifru.ifru_buffer.length = len; +} + +void * +ifr_data_get_ptr(void *ifrp) +{ + union ifreq_union *ifrup; + + ifrup = ifrp; +#ifdef COMPAT_FREEBSD32 + if (SV_CURPROC_FLAG(SV_ILP32)) + return ((void *)(uintptr_t) + ifrup->ifr32.ifr_ifru.ifru_data); +#endif + return (ifrup->ifr.ifr_ifru.ifru_data); +} + /* * Hardware specific interface ioctls. */ @@ -2366,12 +2486,12 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, struct thread *td) else { /* space for terminating nul */ descrlen = strlen(ifp->if_description) + 1; - if (ifr->ifr_buffer.length < descrlen) - ifr->ifr_buffer.buffer = NULL; + if (ifr_buffer_get_length(td, ifr) < descrlen) + ifr_buffer_set_buffer_null(td, ifr); else error = copyout(ifp->if_description, - ifr->ifr_buffer.buffer, descrlen); - ifr->ifr_buffer.length = descrlen; + ifr_buffer_get_buffer(td, ifr), descrlen); + ifr_buffer_set_length(td, ifr, descrlen); } sx_sunlock(&ifdescr_sx); break; @@ -2387,15 +2507,15 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, struct thread *td) * length parameter is supposed to count the * terminating nul in. */ - if (ifr->ifr_buffer.length > ifdescr_maxlen) + if (ifr_buffer_get_length(td, ifr) > ifdescr_maxlen) return (ENAMETOOLONG); - else if (ifr->ifr_buffer.length == 0) + else if (ifr_buffer_get_length(td, ifr) == 0) descrbuf = NULL; else { - descrbuf = malloc(ifr->ifr_buffer.length, M_IFDESCR, - M_WAITOK | M_ZERO); - error = copyin(ifr->ifr_buffer.buffer, descrbuf, - ifr->ifr_buffer.length - 1); + descrbuf = malloc(ifr_buffer_get_length(td, ifr), + M_IFDESCR, M_WAITOK | M_ZERO); + error = copyin(ifr_buffer_get_buffer(td, ifr), descrbuf, + ifr_buffer_get_length(td, ifr) - 1); if (error) { free(descrbuf, M_IFDESCR); break; @@ -2487,7 +2607,8 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, struct thread *td) error = priv_check(td, PRIV_NET_SETIFNAME); if (error) return (error); - error = copyinstr(ifr->ifr_data, new_name, IFNAMSIZ, NULL); + error = copyinstr(ifr_data_get_ptr(ifr), new_name, IFNAMSIZ, + NULL); if (error != 0) return (error); if (new_name[0] == '\0') @@ -2814,8 +2935,8 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td) error = priv_check(td, PRIV_NET_IFCREATE); if (error == 0) error = if_clone_create(ifr->ifr_name, - sizeof(ifr->ifr_name), - cmd == SIOCIFCREATE2 ? ifr->ifr_data : NULL); + sizeof(ifr->ifr_name), cmd == SIOCIFCREATE2 ? + ifr_data_get_ptr(ifr) : NULL); CURVNET_RESTORE(); return (error); case SIOCIFDESTROY: @@ -3545,7 +3666,6 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len) case IFT_ETHER: case IFT_FDDI: case IFT_XETHER: - case IFT_ISO88025: case IFT_L2VLAN: case IFT_BRIDGE: case IFT_ARCNET: diff --git a/freebsd/sys/net/if_arcsubr.c b/freebsd/sys/net/if_arcsubr.c index 171085ec..9f0a03d5 100644 --- a/freebsd/sys/net/if_arcsubr.c +++ b/freebsd/sys/net/if_arcsubr.c @@ -683,12 +683,7 @@ arc_ioctl(struct ifnet *ifp, u_long command, caddr_t data) break; case SIOCGIFADDR: - { - struct sockaddr *sa; - - sa = (struct sockaddr *) &ifr->ifr_data; - *(u_int8_t *)sa->sa_data = ARC_LLADDR(ifp); - } + ifr->ifr_addr.sa_data[0] = ARC_LLADDR(ifp); break; case SIOCADDMULTI: diff --git a/freebsd/sys/net/if_bridge.c b/freebsd/sys/net/if_bridge.c index d345c0f1..248cd45a 100644 --- a/freebsd/sys/net/if_bridge.c +++ b/freebsd/sys/net/if_bridge.c @@ -3178,7 +3178,8 @@ bridge_pfil(struct mbuf **mp, struct ifnet *bifp, struct ifnet *ifp, int dir) if (PFIL_HOOKED(&V_link_pfil_hook) && V_pfil_ipfw != 0 && dir == PFIL_OUT && ifp != NULL) { - error = pfil_run_hooks(&V_link_pfil_hook, mp, ifp, dir, NULL); + error = pfil_run_hooks(&V_link_pfil_hook, mp, ifp, dir, 0, + NULL); if (*mp == NULL || error != 0) /* packet consumed by filter */ return (error); @@ -3230,21 +3231,21 @@ bridge_pfil(struct mbuf **mp, struct ifnet *bifp, struct ifnet *ifp, int dir) */ if (V_pfil_bridge && dir == PFIL_OUT && bifp != NULL) error = pfil_run_hooks(&V_inet_pfil_hook, mp, bifp, - dir, NULL); + dir, 0, NULL); if (*mp == NULL || error != 0) /* filter may consume */ break; if (V_pfil_member && ifp != NULL) error = pfil_run_hooks(&V_inet_pfil_hook, mp, ifp, - dir, NULL); + dir, 0, NULL); if (*mp == NULL || error != 0) /* filter may consume */ break; if (V_pfil_bridge && dir == PFIL_IN && bifp != NULL) error = pfil_run_hooks(&V_inet_pfil_hook, mp, bifp, - dir, NULL); + dir, 0, NULL); if (*mp == NULL || error != 0) /* filter may consume */ break; @@ -3284,21 +3285,21 @@ bridge_pfil(struct mbuf **mp, struct ifnet *bifp, struct ifnet *ifp, int dir) case ETHERTYPE_IPV6: if (V_pfil_bridge && dir == PFIL_OUT && bifp != NULL) error = pfil_run_hooks(&V_inet6_pfil_hook, mp, bifp, - dir, NULL); + dir, 0, NULL); if (*mp == NULL || error != 0) /* filter may consume */ break; if (V_pfil_member && ifp != NULL) error = pfil_run_hooks(&V_inet6_pfil_hook, mp, ifp, - dir, NULL); + dir, 0, NULL); if (*mp == NULL || error != 0) /* filter may consume */ break; if (V_pfil_bridge && dir == PFIL_IN && bifp != NULL) error = pfil_run_hooks(&V_inet6_pfil_hook, mp, bifp, - dir, NULL); + dir, 0, NULL); break; #endif default: diff --git a/freebsd/sys/net/if_enc.c b/freebsd/sys/net/if_enc.c index 34022795..8ca8aa4d 100644 --- a/freebsd/sys/net/if_enc.c +++ b/freebsd/sys/net/if_enc.c @@ -303,7 +303,7 @@ enc_hhook(int32_t hhook_type, int32_t hhook_id, void *udata, void *ctx_data, /* Make a packet looks like it was received on enc(4) */ rcvif = (*ctx->mp)->m_pkthdr.rcvif; (*ctx->mp)->m_pkthdr.rcvif = ifp; - if (pfil_run_hooks(ph, ctx->mp, ifp, pdir, ctx->inp) != 0 || + if (pfil_run_hooks(ph, ctx->mp, ifp, pdir, 0, ctx->inp) != 0 || *ctx->mp == NULL) { *ctx->mp = NULL; /* consumed by filter */ return (EACCES); diff --git a/freebsd/sys/net/if_ethersubr.c b/freebsd/sys/net/if_ethersubr.c index 24f11436..78ff2385 100644 --- a/freebsd/sys/net/if_ethersubr.c +++ b/freebsd/sys/net/if_ethersubr.c @@ -49,6 +49,7 @@ #include <sys/malloc.h> #include <sys/module.h> #include <sys/mbuf.h> +#include <sys/priv.h> #include <sys/random.h> #include <sys/socket.h> #include <sys/sockio.h> @@ -313,7 +314,13 @@ ether_output(struct ifnet *ifp, struct mbuf *m, if (lle == NULL) { /* if we lookup, keep cache */ addref = 1; - } + } else + /* + * Notify LLE code that + * the entry was used + * by datapath. + */ + llentry_mark_used(lle); } if (lle != NULL) { phdr = lle->r_linkdata; @@ -433,6 +440,19 @@ bad: if (m != NULL) return ether_output_frame(ifp, m); } +static bool +ether_set_pcp(struct mbuf **mp, struct ifnet *ifp, uint8_t pcp) +{ + struct ether_header *eh; + + eh = mtod(*mp, struct ether_header *); + if (ntohs(eh->ether_type) == ETHERTYPE_VLAN || + ether_8021q_frame(mp, ifp, ifp, 0, pcp)) + return (true); + if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); + return (false); +} + /* * Ethernet link layer output routine to send a raw frame to the device. * @@ -442,12 +462,17 @@ bad: if (m != NULL) int ether_output_frame(struct ifnet *ifp, struct mbuf *m) { - int i; + int error; + uint8_t pcp; - if (PFIL_HOOKED(&V_link_pfil_hook)) { - i = pfil_run_hooks(&V_link_pfil_hook, &m, ifp, PFIL_OUT, NULL); + pcp = ifp->if_pcp; + if (pcp != IFNET_PCP_NONE && !ether_set_pcp(&m, ifp, pcp)) + return (0); - if (i != 0) + if (PFIL_HOOKED(&V_link_pfil_hook)) { + error = pfil_run_hooks(&V_link_pfil_hook, &m, ifp, + PFIL_OUT, 0, NULL); + if (error != 0) return (EACCES); if (m == NULL) @@ -778,7 +803,8 @@ ether_demux(struct ifnet *ifp, struct mbuf *m) /* Do not grab PROMISC frames in case we are re-entered. */ if (PFIL_HOOKED(&V_link_pfil_hook) && !(m->m_flags & M_PROMISC)) { - i = pfil_run_hooks(&V_link_pfil_hook, &m, ifp, PFIL_IN, NULL); + i = pfil_run_hooks(&V_link_pfil_hook, &m, ifp, PFIL_IN, 0, + NULL); if (i != 0 || m == NULL) return; @@ -1084,13 +1110,8 @@ ether_ioctl(struct ifnet *ifp, u_long command, caddr_t data) break; case SIOCGIFADDR: - { - struct sockaddr *sa; - - sa = (struct sockaddr *) & ifr->ifr_data; - bcopy(IF_LLADDR(ifp), - (caddr_t) sa->sa_data, ETHER_ADDR_LEN); - } + bcopy(IF_LLADDR(ifp), &ifr->ifr_addr.sa_data[0], + ETHER_ADDR_LEN); break; case SIOCSIFMTU: @@ -1103,6 +1124,22 @@ ether_ioctl(struct ifnet *ifp, u_long command, caddr_t data) ifp->if_mtu = ifr->ifr_mtu; } break; + + case SIOCSLANPCP: + error = priv_check(curthread, PRIV_NET_SETLANPCP); + if (error != 0) + break; + if (ifr->ifr_lan_pcp > 7 && + ifr->ifr_lan_pcp != IFNET_PCP_NONE) + error = EINVAL; + else + ifp->if_pcp = ifr->ifr_lan_pcp; + break; + + case SIOCGLANPCP: + ifr->ifr_lan_pcp = ifp->if_pcp; + break; + default: error = EINVAL; /* XXX netbsd has ENOTTY??? */ break; @@ -1251,5 +1288,86 @@ ether_vlanencap(struct mbuf *m, uint16_t tag) return (m); } +static SYSCTL_NODE(_net_link, IFT_L2VLAN, vlan, CTLFLAG_RW, 0, + "IEEE 802.1Q VLAN"); +static SYSCTL_NODE(_net_link_vlan, PF_LINK, link, CTLFLAG_RW, 0, + "for consistency"); + +static VNET_DEFINE(int, soft_pad); +#define V_soft_pad VNET(soft_pad) +SYSCTL_INT(_net_link_vlan, OID_AUTO, soft_pad, CTLFLAG_RW | CTLFLAG_VNET, + &VNET_NAME(soft_pad), 0, + "pad short frames before tagging"); + +/* + * For now, make preserving PCP via an mbuf tag optional, as it increases + * per-packet memory allocations and frees. In the future, it would be + * preferable to reuse ether_vtag for this, or similar. + */ +int vlan_mtag_pcp = 0; +SYSCTL_INT(_net_link_vlan, OID_AUTO, mtag_pcp, CTLFLAG_RW, + &vlan_mtag_pcp, 0, + "Retain VLAN PCP information as packets are passed up the stack"); + +bool +ether_8021q_frame(struct mbuf **mp, struct ifnet *ife, struct ifnet *p, + uint16_t vid, uint8_t pcp) +{ + struct m_tag *mtag; + int n; + uint16_t tag; + static const char pad[8]; /* just zeros */ + + /* + * Pad the frame to the minimum size allowed if told to. + * This option is in accord with IEEE Std 802.1Q, 2003 Ed., + * paragraph C.4.4.3.b. It can help to work around buggy + * bridges that violate paragraph C.4.4.3.a from the same + * document, i.e., fail to pad short frames after untagging. + * E.g., a tagged frame 66 bytes long (incl. FCS) is OK, but + * untagging it will produce a 62-byte frame, which is a runt + * and requires padding. There are VLAN-enabled network + * devices that just discard such runts instead or mishandle + * them somehow. + */ + if (V_soft_pad && p->if_type == IFT_ETHER) { + for (n = ETHERMIN + ETHER_HDR_LEN - (*mp)->m_pkthdr.len; + n > 0; n -= sizeof(pad)) { + if (!m_append(*mp, min(n, sizeof(pad)), pad)) + break; + } + if (n > 0) { + m_freem(*mp); + *mp = NULL; + if_printf(ife, "cannot pad short frame"); + return (false); + } + } + + /* + * If underlying interface can do VLAN tag insertion itself, + * just pass the packet along. However, we need some way to + * tell the interface where the packet came from so that it + * knows how to find the VLAN tag to use, so we attach a + * packet tag that holds it. + */ + if (vlan_mtag_pcp && (mtag = m_tag_locate(*mp, MTAG_8021Q, + MTAG_8021Q_PCP_OUT, NULL)) != NULL) + tag = EVL_MAKETAG(vid, *(uint8_t *)(mtag + 1), 0); + else + tag = EVL_MAKETAG(vid, pcp, 0); + if (p->if_capenable & IFCAP_VLAN_HWTAGGING) { + (*mp)->m_pkthdr.ether_vtag = tag; + (*mp)->m_flags |= M_VLANTAG; + } else { + *mp = ether_vlanencap(*mp, tag); + if (*mp == NULL) { + if_printf(ife, "unable to prepend 802.1Q header"); + return (false); + } + } + return (true); +} + DECLARE_MODULE(ether, ether_mod, SI_SUB_INIT_IF, SI_ORDER_ANY); MODULE_VERSION(ether, 1); diff --git a/freebsd/sys/net/if_fddisubr.c b/freebsd/sys/net/if_fddisubr.c index 3c159a22..c011346e 100644 --- a/freebsd/sys/net/if_fddisubr.c +++ b/freebsd/sys/net/if_fddisubr.c @@ -560,14 +560,9 @@ fddi_ioctl (ifp, command, data) break; } break; - case SIOCGIFADDR: { - struct sockaddr *sa; - - sa = (struct sockaddr *) & ifr->ifr_data; - bcopy(IF_LLADDR(ifp), - (caddr_t) sa->sa_data, FDDI_ADDR_LEN); - - } + case SIOCGIFADDR: + bcopy(IF_LLADDR(ifp), &ifr->ifr_addr.sa_data[0], + FDDI_ADDR_LEN); break; case SIOCSIFMTU: /* diff --git a/freebsd/sys/net/if_fwsubr.c b/freebsd/sys/net/if_fwsubr.c index f9552838..3b39b841 100644 --- a/freebsd/sys/net/if_fwsubr.c +++ b/freebsd/sys/net/if_fwsubr.c @@ -664,13 +664,8 @@ firewire_ioctl(struct ifnet *ifp, u_long command, caddr_t data) break; case SIOCGIFADDR: - { - struct sockaddr *sa; - - sa = (struct sockaddr *) & ifr->ifr_data; - bcopy(&IFP2FWC(ifp)->fc_hwaddr, - (caddr_t) sa->sa_data, sizeof(struct fw_hwaddr)); - } + bcopy(&IFP2FWC(ifp)->fc_hwaddr, &ifr->ifr_addr.sa_data[0], + sizeof(struct fw_hwaddr)); break; case SIOCSIFMTU: diff --git a/freebsd/sys/net/if_gif.c b/freebsd/sys/net/if_gif.c index bd33ab75..6a90538a 100644 --- a/freebsd/sys/net/if_gif.c +++ b/freebsd/sys/net/if_gif.c @@ -904,12 +904,14 @@ gif_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) break; case GIFGOPTS: options = sc->gif_options; - error = copyout(&options, ifr->ifr_data, sizeof(options)); + error = copyout(&options, ifr_data_get_ptr(ifr), + sizeof(options)); break; case GIFSOPTS: if ((error = priv_check(curthread, PRIV_NET_GIF)) != 0) break; - error = copyin(ifr->ifr_data, &options, sizeof(options)); + error = copyin(ifr_data_get_ptr(ifr), &options, + sizeof(options)); if (error) break; if (options & ~GIF_OPTMASK) diff --git a/freebsd/sys/net/if_gre.c b/freebsd/sys/net/if_gre.c index dd9ba8f3..0bff9bc9 100644 --- a/freebsd/sys/net/if_gre.c +++ b/freebsd/sys/net/if_gre.c @@ -461,7 +461,8 @@ gre_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) case GRESKEY: if ((error = priv_check(curthread, PRIV_NET_GRE)) != 0) break; - if ((error = copyin(ifr->ifr_data, &opt, sizeof(opt))) != 0) + if ((error = copyin(ifr_data_get_ptr(ifr), &opt, + sizeof(opt))) != 0) break; if (sc->gre_key != opt) { GRE_WLOCK(sc); @@ -471,13 +472,14 @@ gre_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) } break; case GREGKEY: - error = copyout(&sc->gre_key, ifr->ifr_data, + error = copyout(&sc->gre_key, ifr_data_get_ptr(ifr), sizeof(sc->gre_key)); break; case GRESOPTS: if ((error = priv_check(curthread, PRIV_NET_GRE)) != 0) break; - if ((error = copyin(ifr->ifr_data, &opt, sizeof(opt))) != 0) + if ((error = copyin(ifr_data_get_ptr(ifr), &opt, + sizeof(opt))) != 0) break; if (opt & ~GRE_OPTMASK) error = EINVAL; @@ -492,7 +494,7 @@ gre_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) break; case GREGOPTS: - error = copyout(&sc->gre_options, ifr->ifr_data, + error = copyout(&sc->gre_options, ifr_data_get_ptr(ifr), sizeof(sc->gre_options)); break; default: diff --git a/freebsd/sys/net/if_ipsec.c b/freebsd/sys/net/if_ipsec.c index 13d3f44b..91afe2ab 100644 --- a/freebsd/sys/net/if_ipsec.c +++ b/freebsd/sys/net/if_ipsec.c @@ -694,12 +694,12 @@ ipsec_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) break; case IPSECGREQID: reqid = sc->reqid; - error = copyout(&reqid, ifr->ifr_data, sizeof(reqid)); + error = copyout(&reqid, ifr_data_get_ptr(ifr), sizeof(reqid)); break; case IPSECSREQID: if ((error = priv_check(curthread, PRIV_NET_SETIFCAP)) != 0) break; - error = copyin(ifr->ifr_data, &reqid, sizeof(reqid)); + error = copyin(ifr_data_get_ptr(ifr), &reqid, sizeof(reqid)); if (error != 0) break; error = ipsec_set_reqid(ifp, reqid); diff --git a/freebsd/sys/net/if_iso88025subr.c b/freebsd/sys/net/if_iso88025subr.c deleted file mode 100644 index cba09bf5..00000000 --- a/freebsd/sys/net/if_iso88025subr.c +++ /dev/null @@ -1,701 +0,0 @@ -#include <machine/rtems-bsd-kernel-space.h> - -/*- - * SPDX-License-Identifier: BSD-2-Clause-FreeBSD - * - * Copyright (c) 1998, Larry Lile - * All rights reserved. - * - * For latest sources and information on this driver, please - * go to http://anarchy.stdio.com. - * - * Questions, comments or suggestions should be directed to - * Larry Lile <lile@stdio.com>. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice unmodified, this list of conditions, and the following - * disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - * - */ - -/* - * - * General ISO 802.5 (Token Ring) support routines - * - */ - -#include <rtems/bsd/local/opt_inet.h> -#include <rtems/bsd/local/opt_inet6.h> - -#include <sys/param.h> -#include <sys/systm.h> -#include <sys/kernel.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/module.h> -#include <sys/socket.h> -#include <sys/sockio.h> - -#include <net/if.h> -#include <net/if_var.h> -#include <net/if_arp.h> -#include <net/if_dl.h> -#include <net/if_llc.h> -#include <net/if_types.h> -#include <net/if_llatbl.h> - -#include <net/ethernet.h> -#include <net/netisr.h> -#include <net/route.h> -#include <net/bpf.h> -#include <net/iso88025.h> - -#if defined(INET) || defined(INET6) -#include <netinet/in.h> -#include <netinet/in_var.h> -#include <netinet/if_ether.h> -#endif -#ifdef INET6 -#include <netinet6/nd6.h> -#endif - -#include <security/mac/mac_framework.h> - -static const u_char iso88025_broadcastaddr[ISO88025_ADDR_LEN] = - { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; - -static int iso88025_resolvemulti (struct ifnet *, struct sockaddr **, - struct sockaddr *); - -#define senderr(e) do { error = (e); goto bad; } while (0) - -/* - * Perform common duties while attaching to interface list - */ -void -iso88025_ifattach(struct ifnet *ifp, const u_int8_t *lla, int bpf) -{ - struct ifaddr *ifa; - struct sockaddr_dl *sdl; - - ifa = NULL; - - ifp->if_type = IFT_ISO88025; - ifp->if_addrlen = ISO88025_ADDR_LEN; - ifp->if_hdrlen = ISO88025_HDR_LEN; - - if_attach(ifp); /* Must be called before additional assignments */ - - ifp->if_output = iso88025_output; - ifp->if_input = iso88025_input; - ifp->if_resolvemulti = iso88025_resolvemulti; - ifp->if_broadcastaddr = iso88025_broadcastaddr; - - if (ifp->if_baudrate == 0) - ifp->if_baudrate = TR_16MBPS; /* 16Mbit should be a safe default */ - if (ifp->if_mtu == 0) - ifp->if_mtu = ISO88025_DEFAULT_MTU; - - ifa = ifp->if_addr; - KASSERT(ifa != NULL, ("%s: no lladdr!\n", __func__)); - - sdl = (struct sockaddr_dl *)ifa->ifa_addr; - sdl->sdl_type = IFT_ISO88025; - sdl->sdl_alen = ifp->if_addrlen; - bcopy(lla, LLADDR(sdl), ifp->if_addrlen); - - if (bpf) - bpfattach(ifp, DLT_IEEE802, ISO88025_HDR_LEN); - - return; -} - -/* - * Perform common duties while detaching a Token Ring interface - */ -void -iso88025_ifdetach(ifp, bpf) - struct ifnet *ifp; - int bpf; -{ - - if (bpf) - bpfdetach(ifp); - - if_detach(ifp); - - return; -} - -int -iso88025_ioctl(struct ifnet *ifp, u_long command, caddr_t data) -{ - struct ifaddr *ifa; - struct ifreq *ifr; - int error; - - ifa = (struct ifaddr *) data; - ifr = (struct ifreq *) data; - error = 0; - - switch (command) { - case SIOCSIFADDR: - ifp->if_flags |= IFF_UP; - - switch (ifa->ifa_addr->sa_family) { -#ifdef INET - case AF_INET: - ifp->if_init(ifp->if_softc); /* before arpwhohas */ - arp_ifinit(ifp, ifa); - break; -#endif /* INET */ - default: - ifp->if_init(ifp->if_softc); - break; - } - break; - - case SIOCGIFADDR: { - struct sockaddr *sa; - - sa = (struct sockaddr *) & ifr->ifr_data; - bcopy(IF_LLADDR(ifp), - (caddr_t) sa->sa_data, ISO88025_ADDR_LEN); - } - break; - - case SIOCSIFMTU: - /* - * Set the interface MTU. - */ - if (ifr->ifr_mtu > ISO88025_MAX_MTU) { - error = EINVAL; - } else { - ifp->if_mtu = ifr->ifr_mtu; - } - break; - default: - error = EINVAL; /* XXX netbsd has ENOTTY??? */ - break; - } - - return (error); -} - -/* - * ISO88025 encapsulation - */ -int -iso88025_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, - struct route *ro) -{ - u_int16_t snap_type = 0; - int loop_copy = 0, error = 0, rif_len = 0; - u_char edst[ISO88025_ADDR_LEN]; - struct iso88025_header *th; - struct iso88025_header gen_th; - struct sockaddr_dl *sdl = NULL; - struct rtentry *rt0 = NULL; - int is_gw = 0; - - if (ro != NULL) - is_gw = (ro->ro_flags & RT_HAS_GW) != 0; -#ifdef MAC - error = mac_ifnet_check_transmit(ifp, m); - if (error) - senderr(error); -#endif - - if (ifp->if_flags & IFF_MONITOR) - senderr(ENETDOWN); - if (!((ifp->if_flags & IFF_UP) && - (ifp->if_drv_flags & IFF_DRV_RUNNING))) - senderr(ENETDOWN); - getmicrotime(&ifp->if_lastchange); - - /* Calculate routing info length based on arp table entry */ - /* XXX any better way to do this ? */ - - if (rt0 && (sdl = (struct sockaddr_dl *)rt0->rt_gateway)) - if (SDL_ISO88025(sdl)->trld_rcf != 0) - rif_len = TR_RCF_RIFLEN(SDL_ISO88025(sdl)->trld_rcf); - - /* Generate a generic 802.5 header for the packet */ - gen_th.ac = TR_AC; - gen_th.fc = TR_LLC_FRAME; - (void)memcpy((caddr_t)gen_th.iso88025_shost, IF_LLADDR(ifp), - ISO88025_ADDR_LEN); - if (rif_len) { - gen_th.iso88025_shost[0] |= TR_RII; - if (rif_len > 2) { - gen_th.rcf = SDL_ISO88025(sdl)->trld_rcf; - (void)memcpy((caddr_t)gen_th.rd, - (caddr_t)SDL_ISO88025(sdl)->trld_route, - rif_len - 2); - } - } - - switch (dst->sa_family) { -#ifdef INET - case AF_INET: - error = arpresolve(ifp, is_gw, m, dst, edst, NULL, NULL); - if (error) - return (error == EWOULDBLOCK ? 0 : error); - snap_type = ETHERTYPE_IP; - break; - case AF_ARP: - { - struct arphdr *ah; - ah = mtod(m, struct arphdr *); - ah->ar_hrd = htons(ARPHRD_IEEE802); - - loop_copy = -1; /* if this is for us, don't do it */ - - switch(ntohs(ah->ar_op)) { - case ARPOP_REVREQUEST: - case ARPOP_REVREPLY: - snap_type = ETHERTYPE_REVARP; - break; - case ARPOP_REQUEST: - case ARPOP_REPLY: - default: - snap_type = ETHERTYPE_ARP; - break; - } - - if (m->m_flags & M_BCAST) - bcopy(ifp->if_broadcastaddr, edst, ISO88025_ADDR_LEN); - else - bcopy(ar_tha(ah), edst, ISO88025_ADDR_LEN); - - } - break; -#endif /* INET */ -#ifdef INET6 - case AF_INET6: - error = nd6_resolve(ifp, is_gw, m, dst, edst, NULL, NULL); - if (error) - return (error == EWOULDBLOCK ? 0 : error); - snap_type = ETHERTYPE_IPV6; - break; -#endif /* INET6 */ - case AF_UNSPEC: - { - const struct iso88025_sockaddr_data *sd; - /* - * For AF_UNSPEC sockaddr.sa_data must contain all of the - * mac information needed to send the packet. This allows - * full mac, llc, and source routing function to be controlled. - * llc and source routing information must already be in the - * mbuf provided, ac/fc are set in sa_data. sockaddr.sa_data - * should be an iso88025_sockaddr_data structure see iso88025.h - */ - loop_copy = -1; - sd = (const struct iso88025_sockaddr_data *)dst->sa_data; - gen_th.ac = sd->ac; - gen_th.fc = sd->fc; - (void)memcpy(edst, sd->ether_dhost, ISO88025_ADDR_LEN); - (void)memcpy(gen_th.iso88025_shost, sd->ether_shost, - ISO88025_ADDR_LEN); - rif_len = 0; - break; - } - default: - if_printf(ifp, "can't handle af%d\n", dst->sa_family); - senderr(EAFNOSUPPORT); - break; - } - - /* - * Add LLC header. - */ - if (snap_type != 0) { - struct llc *l; - M_PREPEND(m, LLC_SNAPFRAMELEN, M_NOWAIT); - if (m == NULL) - senderr(ENOBUFS); - l = mtod(m, struct llc *); - l->llc_control = LLC_UI; - l->llc_dsap = l->llc_ssap = LLC_SNAP_LSAP; - l->llc_snap.org_code[0] = - l->llc_snap.org_code[1] = - l->llc_snap.org_code[2] = 0; - l->llc_snap.ether_type = htons(snap_type); - } - - /* - * Add local net header. If no space in first mbuf, - * allocate another. - */ - M_PREPEND(m, ISO88025_HDR_LEN + rif_len, M_NOWAIT); - if (m == NULL) - senderr(ENOBUFS); - th = mtod(m, struct iso88025_header *); - bcopy((caddr_t)edst, (caddr_t)&gen_th.iso88025_dhost, ISO88025_ADDR_LEN); - - /* Copy as much of the generic header as is needed into the mbuf */ - memcpy(th, &gen_th, ISO88025_HDR_LEN + rif_len); - - /* - * If a simplex interface, and the packet is being sent to our - * Ethernet address or a broadcast address, loopback a copy. - * XXX To make a simplex device behave exactly like a duplex - * device, we should copy in the case of sending to our own - * ethernet address (thus letting the original actually appear - * on the wire). However, we don't do that here for security - * reasons and compatibility with the original behavior. - */ - if ((ifp->if_flags & IFF_SIMPLEX) && (loop_copy != -1)) { - if ((m->m_flags & M_BCAST) || (loop_copy > 0)) { - struct mbuf *n; - n = m_copym(m, 0, M_COPYALL, M_NOWAIT); - (void) if_simloop(ifp, n, dst->sa_family, - ISO88025_HDR_LEN); - } else if (bcmp(th->iso88025_dhost, th->iso88025_shost, - ETHER_ADDR_LEN) == 0) { - (void) if_simloop(ifp, m, dst->sa_family, - ISO88025_HDR_LEN); - return(0); /* XXX */ - } - } - - IFQ_HANDOFF_ADJ(ifp, m, ISO88025_HDR_LEN + LLC_SNAPFRAMELEN, error); - if (error) { - printf("iso88025_output: packet dropped QFULL.\n"); - if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - } - return (error); - -bad: - if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - if (m) - m_freem(m); - return (error); -} - -/* - * ISO 88025 de-encapsulation - */ -void -iso88025_input(ifp, m) - struct ifnet *ifp; - struct mbuf *m; -{ - struct iso88025_header *th; - struct llc *l; - int isr; - int mac_hdr_len; - - /* - * Do consistency checks to verify assumptions - * made by code past this point. - */ - if ((m->m_flags & M_PKTHDR) == 0) { - if_printf(ifp, "discard frame w/o packet header\n"); - if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); - m_freem(m); - return; - } - if (m->m_pkthdr.rcvif == NULL) { - if_printf(ifp, "discard frame w/o interface pointer\n"); - if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); - m_freem(m); - return; - } - - m = m_pullup(m, ISO88025_HDR_LEN); - if (m == NULL) { - if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); - goto dropanyway; - } - th = mtod(m, struct iso88025_header *); - - /* - * Discard packet if interface is not up. - */ - if (!((ifp->if_flags & IFF_UP) && - (ifp->if_drv_flags & IFF_DRV_RUNNING))) - goto dropanyway; - - /* - * Give bpf a chance at the packet. - */ - BPF_MTAP(ifp, m); - - /* - * Interface marked for monitoring; discard packet. - */ - if (ifp->if_flags & IFF_MONITOR) { - m_freem(m); - return; - } - -#ifdef MAC - mac_ifnet_create_mbuf(ifp, m); -#endif - - /* - * Update interface statistics. - */ - if_inc_counter(ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len); - getmicrotime(&ifp->if_lastchange); - - /* - * Discard non local unicast packets when interface - * is in promiscuous mode. - */ - if ((ifp->if_flags & IFF_PROMISC) && - ((th->iso88025_dhost[0] & 1) == 0) && - (bcmp(IF_LLADDR(ifp), (caddr_t) th->iso88025_dhost, - ISO88025_ADDR_LEN) != 0)) - goto dropanyway; - - /* - * Set mbuf flags for bcast/mcast. - */ - if (th->iso88025_dhost[0] & 1) { - if (bcmp(iso88025_broadcastaddr, th->iso88025_dhost, - ISO88025_ADDR_LEN) == 0) - m->m_flags |= M_BCAST; - else - m->m_flags |= M_MCAST; - if_inc_counter(ifp, IFCOUNTER_IMCASTS, 1); - } - - mac_hdr_len = ISO88025_HDR_LEN; - /* Check for source routing info */ - if (th->iso88025_shost[0] & TR_RII) - mac_hdr_len += TR_RCF_RIFLEN(th->rcf); - - /* Strip off ISO88025 header. */ - m_adj(m, mac_hdr_len); - - m = m_pullup(m, LLC_SNAPFRAMELEN); - if (m == NULL) { - if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); - goto dropanyway; - } - l = mtod(m, struct llc *); - - switch (l->llc_dsap) { - case LLC_SNAP_LSAP: { - u_int16_t type; - if ((l->llc_control != LLC_UI) || - (l->llc_ssap != LLC_SNAP_LSAP)) { - if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1); - goto dropanyway; - } - - if (l->llc_snap.org_code[0] != 0 || - l->llc_snap.org_code[1] != 0 || - l->llc_snap.org_code[2] != 0) { - if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1); - goto dropanyway; - } - - type = ntohs(l->llc_snap.ether_type); - m_adj(m, LLC_SNAPFRAMELEN); - switch (type) { -#ifdef INET - case ETHERTYPE_IP: - th->iso88025_shost[0] &= ~(TR_RII); - isr = NETISR_IP; - break; - - case ETHERTYPE_ARP: - if (ifp->if_flags & IFF_NOARP) - goto dropanyway; - isr = NETISR_ARP; - break; -#endif /* INET */ -#ifdef INET6 - case ETHERTYPE_IPV6: - th->iso88025_shost[0] &= ~(TR_RII); - isr = NETISR_IPV6; - break; -#endif /* INET6 */ - default: - printf("iso88025_input: unexpected llc_snap ether_type 0x%02x\n", type); - if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1); - goto dropanyway; - } - break; - } -#ifdef ISO - case LLC_ISO_LSAP: - switch (l->llc_control) { - case LLC_UI: - if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1); - goto dropanyway; - break; - case LLC_XID: - case LLC_XID_P: - if(m->m_len < ISO88025_ADDR_LEN) - goto dropanyway; - l->llc_window = 0; - l->llc_fid = 9; - l->llc_class = 1; - l->llc_dsap = l->llc_ssap = 0; - /* Fall through to */ - case LLC_TEST: - case LLC_TEST_P: - { - struct sockaddr sa; - struct iso88025_sockaddr_data *th2; - int i; - u_char c; - - c = l->llc_dsap; - - if (th->iso88025_shost[0] & TR_RII) { /* XXX */ - printf("iso88025_input: dropping source routed LLC_TEST\n"); - goto dropanyway; - } - l->llc_dsap = l->llc_ssap; - l->llc_ssap = c; - if (m->m_flags & (M_BCAST | M_MCAST)) - bcopy((caddr_t)IF_LLADDR(ifp), - (caddr_t)th->iso88025_dhost, - ISO88025_ADDR_LEN); - sa.sa_family = AF_UNSPEC; - sa.sa_len = sizeof(sa); - th2 = (struct iso88025_sockaddr_data *)sa.sa_data; - for (i = 0; i < ISO88025_ADDR_LEN; i++) { - th2->ether_shost[i] = c = th->iso88025_dhost[i]; - th2->ether_dhost[i] = th->iso88025_dhost[i] = - th->iso88025_shost[i]; - th->iso88025_shost[i] = c; - } - th2->ac = TR_AC; - th2->fc = TR_LLC_FRAME; - ifp->if_output(ifp, m, &sa, NULL); - return; - } - default: - printf("iso88025_input: unexpected llc control 0x%02x\n", l->llc_control); - if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1); - goto dropanyway; - break; - } - break; -#endif /* ISO */ - default: - printf("iso88025_input: unknown dsap 0x%x\n", l->llc_dsap); - if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1); - goto dropanyway; - break; - } - - M_SETFIB(m, ifp->if_fib); - netisr_dispatch(isr, m); - return; - -dropanyway: - if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); - if (m) - m_freem(m); - return; -} - -static int -iso88025_resolvemulti (ifp, llsa, sa) - struct ifnet *ifp; - struct sockaddr **llsa; - struct sockaddr *sa; -{ - struct sockaddr_dl *sdl; -#ifdef INET - struct sockaddr_in *sin; -#endif -#ifdef INET6 - struct sockaddr_in6 *sin6; -#endif - u_char *e_addr; - - switch(sa->sa_family) { - case AF_LINK: - /* - * No mapping needed. Just check that it's a valid MC address. - */ - sdl = (struct sockaddr_dl *)sa; - e_addr = LLADDR(sdl); - if ((e_addr[0] & 1) != 1) { - return (EADDRNOTAVAIL); - } - *llsa = NULL; - return (0); - -#ifdef INET - case AF_INET: - sin = (struct sockaddr_in *)sa; - if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { - return (EADDRNOTAVAIL); - } - sdl = link_init_sdl(ifp, *llsa, IFT_ISO88025); - sdl->sdl_alen = ISO88025_ADDR_LEN; - e_addr = LLADDR(sdl); - ETHER_MAP_IP_MULTICAST(&sin->sin_addr, e_addr); - *llsa = (struct sockaddr *)sdl; - return (0); -#endif -#ifdef INET6 - case AF_INET6: - sin6 = (struct sockaddr_in6 *)sa; - if (IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) { - /* - * An IP6 address of 0 means listen to all - * of the Ethernet multicast address used for IP6. - * (This is used for multicast routers.) - */ - ifp->if_flags |= IFF_ALLMULTI; - *llsa = NULL; - return (0); - } - if (!IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr)) { - return (EADDRNOTAVAIL); - } - sdl = link_init_sdl(ifp, *llsa, IFT_ISO88025); - sdl->sdl_alen = ISO88025_ADDR_LEN; - e_addr = LLADDR(sdl); - ETHER_MAP_IPV6_MULTICAST(&sin6->sin6_addr, e_addr); - *llsa = (struct sockaddr *)sdl; - return (0); -#endif - - default: - /* - * Well, the text isn't quite right, but it's the name - * that counts... - */ - return (EAFNOSUPPORT); - } - - return (0); -} - -static moduledata_t iso88025_mod = { - .name = "iso88025", -}; - -DECLARE_MODULE(iso88025, iso88025_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); -MODULE_VERSION(iso88025, 1); diff --git a/freebsd/sys/net/if_llatbl.h b/freebsd/sys/net/if_llatbl.h index fcf93883..9cec4ac7 100644 --- a/freebsd/sys/net/if_llatbl.h +++ b/freebsd/sys/net/if_llatbl.h @@ -149,6 +149,7 @@ typedef void (llt_fill_sa_entry_t)(const struct llentry *, struct sockaddr *); typedef void (llt_free_tbl_t)(struct lltable *); typedef void (llt_link_entry_t)(struct lltable *, struct llentry *); typedef void (llt_unlink_entry_t)(struct llentry *); +typedef void (llt_mark_used_t)(struct llentry *); typedef int (llt_foreach_cb_t)(struct lltable *, struct llentry *, void *); typedef int (llt_foreach_entry_t)(struct lltable *, llt_foreach_cb_t *, void *); @@ -173,6 +174,7 @@ struct lltable { llt_unlink_entry_t *llt_unlink_entry; llt_fill_sa_entry_t *llt_fill_sa_entry; llt_free_tbl_t *llt_free_tbl; + llt_mark_used_t *llt_mark_used; }; MALLOC_DECLARE(M_LLTABLE); @@ -247,6 +249,19 @@ lla_lookup(struct lltable *llt, u_int flags, const struct sockaddr *l3addr) return (llt->llt_lookup(llt, flags, l3addr)); } +/* + * Notify the LLE code that the entry was used by datapath. + */ +static __inline void +llentry_mark_used(struct llentry *lle) +{ + + if (lle->r_skip_req == 0) + return; + if ((lle->r_flags & RLLE_VALID) != 0) + lle->lle_tbl->llt_mark_used(lle); +} + int lla_rt_output(struct rt_msghdr *, struct rt_addrinfo *); #include <sys/eventhandler.h> diff --git a/freebsd/sys/net/if_media.h b/freebsd/sys/net/if_media.h index 13621a10..ecc2b418 100644 --- a/freebsd/sys/net/if_media.h +++ b/freebsd/sys/net/if_media.h @@ -270,7 +270,7 @@ uint64_t ifmedia_baudrate(int); #define IFM_IEEE80211_OFDM27 23 /* OFDM 27Mbps */ /* NB: not enough bits to express MCS fully */ #define IFM_IEEE80211_MCS 24 /* HT MCS rate */ -#define IFM_IEEE80211_VHT 25 /* HT MCS rate */ +#define IFM_IEEE80211_VHT 25 /* VHT MCS rate */ #define IFM_IEEE80211_ADHOC 0x00000100 /* Operate in Adhoc mode */ #define IFM_IEEE80211_HOSTAP 0x00000200 /* Operate in Host AP mode */ diff --git a/freebsd/sys/net/if_spppsubr.c b/freebsd/sys/net/if_spppsubr.c index 306b7e2f..1f85c00f 100644 --- a/freebsd/sys/net/if_spppsubr.c +++ b/freebsd/sys/net/if_spppsubr.c @@ -5060,17 +5060,17 @@ sppp_params(struct sppp *sp, u_long cmd, void *data) if ((spr = malloc(sizeof(struct spppreq), M_TEMP, M_NOWAIT)) == NULL) return (EAGAIN); /* - * ifr->ifr_data is supposed to point to a struct spppreq. + * ifr_data_get_ptr(ifr) is supposed to point to a struct spppreq. * Check the cmd word first before attempting to fetch all the * data. */ - rv = fueword(ifr->ifr_data, &subcmd); + rv = fueword(ifr_data_get_ptr(ifr), &subcmd); if (rv == -1) { rv = EFAULT; goto quit; } - if (copyin((caddr_t)ifr->ifr_data, spr, sizeof(struct spppreq)) != 0) { + if (copyin(ifr_data_get_ptr(ifr), spr, sizeof(struct spppreq)) != 0) { rv = EFAULT; goto quit; } @@ -5107,8 +5107,8 @@ sppp_params(struct sppp *sp, u_long cmd, void *data) * setting it. */ spr->defs.lcp.timeout = sp->lcp.timeout * 1000 / hz; - rv = copyout(spr, (caddr_t)ifr->ifr_data, - sizeof(struct spppreq)); + rv = copyout(spr, ifr_data_get_ptr(ifr), + sizeof(struct spppreq)); break; case (u_long)SPPPIOSDEFS: diff --git a/freebsd/sys/net/if_var.h b/freebsd/sys/net/if_var.h index e22beea7..a131c496 100644 --- a/freebsd/sys/net/if_var.h +++ b/freebsd/sys/net/if_var.h @@ -365,6 +365,9 @@ struct ifnet { if_snd_tag_query_t *if_snd_tag_query; if_snd_tag_free_t *if_snd_tag_free; + /* Ethernet PCP */ + uint8_t if_pcp; + #ifndef __rtems__ /* * Spare fields to be added before branching a stable branch, so @@ -730,6 +733,9 @@ int drbr_enqueue_drv(if_t ifp, struct buf_ring *br, struct mbuf *m); void if_hw_tsomax_common(if_t ifp, struct ifnet_hw_tsomax *); int if_hw_tsomax_update(if_t ifp, struct ifnet_hw_tsomax *); +/* accessors for struct ifreq */ +void *ifr_data_get_ptr(void *ifrp); + #ifdef DEVICE_POLLING enum poll_cmd { POLL_ONLY, POLL_AND_CHECK_STATUS }; diff --git a/freebsd/sys/net/if_vlan.c b/freebsd/sys/net/if_vlan.c index 872b07ad..79294427 100644 --- a/freebsd/sys/net/if_vlan.c +++ b/freebsd/sys/net/if_vlan.c @@ -198,25 +198,7 @@ static struct { {0, NULL} }; -SYSCTL_DECL(_net_link); -static SYSCTL_NODE(_net_link, IFT_L2VLAN, vlan, CTLFLAG_RW, 0, - "IEEE 802.1Q VLAN"); -static SYSCTL_NODE(_net_link_vlan, PF_LINK, link, CTLFLAG_RW, 0, - "for consistency"); - -static VNET_DEFINE(int, soft_pad); -#define V_soft_pad VNET(soft_pad) -SYSCTL_INT(_net_link_vlan, OID_AUTO, soft_pad, CTLFLAG_RW | CTLFLAG_VNET, - &VNET_NAME(soft_pad), 0, "pad short frames before tagging"); - -/* - * For now, make preserving PCP via an mbuf tag optional, as it increases - * per-packet memory allocations and frees. In the future, it would be - * preferable to reuse ether_vtag for this, or similar. - */ -static int vlan_mtag_pcp = 0; -SYSCTL_INT(_net_link_vlan, OID_AUTO, mtag_pcp, CTLFLAG_RW, &vlan_mtag_pcp, 0, - "Retain VLAN PCP information as packets are passed up the stack"); +extern int vlan_mtag_pcp; static const char vlanname[] = "vlan"; static MALLOC_DEFINE(M_VLAN, vlanname, "802.1Q Virtual LAN Interface"); @@ -1173,8 +1155,6 @@ vlan_transmit(struct ifnet *ifp, struct mbuf *m) { struct ifvlan *ifv; struct ifnet *p; - struct m_tag *mtag; - uint16_t tag; int error, len, mcast; VLAN_LOCK_READER; @@ -1203,59 +1183,10 @@ vlan_transmit(struct ifnet *ifp, struct mbuf *m) return (ENETDOWN); } - /* - * Pad the frame to the minimum size allowed if told to. - * This option is in accord with IEEE Std 802.1Q, 2003 Ed., - * paragraph C.4.4.3.b. It can help to work around buggy - * bridges that violate paragraph C.4.4.3.a from the same - * document, i.e., fail to pad short frames after untagging. - * E.g., a tagged frame 66 bytes long (incl. FCS) is OK, but - * untagging it will produce a 62-byte frame, which is a runt - * and requires padding. There are VLAN-enabled network - * devices that just discard such runts instead or mishandle - * them somehow. - */ - if (V_soft_pad && p->if_type == IFT_ETHER) { - static char pad[8]; /* just zeros */ - int n; - - for (n = ETHERMIN + ETHER_HDR_LEN - m->m_pkthdr.len; - n > 0; n -= sizeof(pad)) - if (!m_append(m, min(n, sizeof(pad)), pad)) - break; - - if (n > 0) { - if_printf(ifp, "cannot pad short frame\n"); - if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - VLAN_RUNLOCK(); - m_freem(m); - return (0); - } - } - - /* - * If underlying interface can do VLAN tag insertion itself, - * just pass the packet along. However, we need some way to - * tell the interface where the packet came from so that it - * knows how to find the VLAN tag to use, so we attach a - * packet tag that holds it. - */ - if (vlan_mtag_pcp && (mtag = m_tag_locate(m, MTAG_8021Q, - MTAG_8021Q_PCP_OUT, NULL)) != NULL) - tag = EVL_MAKETAG(ifv->ifv_vid, *(uint8_t *)(mtag + 1), 0); - else - tag = ifv->ifv_tag; - if (p->if_capenable & IFCAP_VLAN_HWTAGGING) { - m->m_pkthdr.ether_vtag = tag; - m->m_flags |= M_VLANTAG; - } else { - m = ether_vlanencap(m, tag); - if (m == NULL) { - if_printf(ifp, "unable to prepend VLAN header\n"); - if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - VLAN_RUNLOCK(); - return (0); - } + if (!ether_8021q_frame(&m, ifp, p, ifv->ifv_vid, ifv->ifv_pcp)) { + if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); + VLAN_RUNLOCK(); + return (0); } /* @@ -1861,12 +1792,8 @@ vlan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) #endif break; case SIOCGIFADDR: - { - struct sockaddr *sa; - - sa = (struct sockaddr *)&ifr->ifr_data; - bcopy(IF_LLADDR(ifp), sa->sa_data, ifp->if_addrlen); - } + bcopy(IF_LLADDR(ifp), &ifr->ifr_addr.sa_data[0], + ifp->if_addrlen); break; case SIOCGIFMEDIA: VLAN_SLOCK(); @@ -1932,7 +1859,7 @@ vlan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) break; } #endif - error = copyin(ifr->ifr_data, &vlr, sizeof(vlr)); + error = copyin(ifr_data_get_ptr(ifr), &vlr, sizeof(vlr)); if (error) break; if (vlr.vlr_parent[0] == '\0') { @@ -1963,7 +1890,7 @@ vlan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) vlr.vlr_tag = ifv->ifv_vid; } VLAN_SUNLOCK(); - error = copyout(&vlr, ifr->ifr_data, sizeof(vlr)); + error = copyout(&vlr, ifr_data_get_ptr(ifr), sizeof(vlr)); break; case SIOCSIFFLAGS: diff --git a/freebsd/sys/net/if_vlan_var.h b/freebsd/sys/net/if_vlan_var.h index d27e5104..b926df80 100644 --- a/freebsd/sys/net/if_vlan_var.h +++ b/freebsd/sys/net/if_vlan_var.h @@ -73,22 +73,8 @@ struct vlanreq { #define SIOCSETVLAN SIOCSIFGENERIC #define SIOCGETVLAN SIOCGIFGENERIC -#define SIOCGVLANPCP _IOWR('i', 152, struct ifreq) /* Get VLAN PCP */ -#define SIOCSVLANPCP _IOW('i', 153, struct ifreq) /* Set VLAN PCP */ - -/* - * Names for 802.1q priorities ("802.1p"). Notice that in this scheme, - * (0 < 1), allowing default 0-tagged traffic to take priority over background - * tagged traffic. - */ -#define IEEE8021Q_PCP_BK 1 /* Background (lowest) */ -#define IEEE8021Q_PCP_BE 0 /* Best effort (default) */ -#define IEEE8021Q_PCP_EE 2 /* Excellent effort */ -#define IEEE8021Q_PCP_CA 3 /* Critical applications */ -#define IEEE8021Q_PCP_VI 4 /* Video, < 100ms latency */ -#define IEEE8021Q_PCP_VO 5 /* Video, < 10ms latency */ -#define IEEE8021Q_PCP_IC 6 /* Internetwork control */ -#define IEEE8021Q_PCP_NC 7 /* Network control (highest) */ +#define SIOCGVLANPCP SIOCGLANPCP /* Get VLAN PCP */ +#define SIOCSVLANPCP SIOCSLANPCP /* Set VLAN PCP */ #ifdef _KERNEL /* diff --git a/freebsd/sys/net/iflib.h b/freebsd/sys/net/iflib.h index 3730f0ea..02319322 100644 --- a/freebsd/sys/net/iflib.h +++ b/freebsd/sys/net/iflib.h @@ -229,7 +229,7 @@ typedef struct if_softc_ctx { * Initialization values for device */ struct if_shared_ctx { - int isc_magic; + unsigned isc_magic; driver_t *isc_driver; bus_size_t isc_q_align; bus_size_t isc_tx_maxsize; diff --git a/freebsd/sys/net/iso88025.h b/freebsd/sys/net/iso88025.h deleted file mode 100644 index 78b2737f..00000000 --- a/freebsd/sys/net/iso88025.h +++ /dev/null @@ -1,176 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-2-Clause-FreeBSD - * - * Copyright (c) 1998, Larry Lile - * All rights reserved. - * - * For latest sources and information on this driver, please - * go to http://anarchy.stdio.com. - * - * Questions, comments or suggestions should be directed to - * Larry Lile <lile@stdio.com>. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice unmodified, this list of conditions, and the following - * disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - * - * Information gathered from tokenring@freebsd, /sys/net/ethernet.h and - * the Mach token ring driver. - */ - -/* - * Fundamental constants relating to iso 802.5 - */ - -#ifndef _NET_ISO88025_H_ -#define _NET_ISO88025_H_ - -/* - * General ISO 802.5 definitions - */ -#define ISO88025_ADDR_LEN 6 -#define ISO88025_CF_LEN 2 -#define ISO88025_HDR_LEN (ISO88025_CF_LEN + (ISO88025_ADDR_LEN * 2)) -#define RCF_LEN 2 -#define RIF_MAX_RD 14 -#define RIF_MAX_LEN 16 - -#define TR_AC 0x10 -#define TR_LLC_FRAME 0x40 - -#define TR_4MBPS 4000000 -#define TR_16MBPS 16000000 -#define TR_100MBPS 100000000 - -/* - * Source routing - */ -#define TR_RII 0x80 -#define TR_RCF_BCST_MASK 0xe000 -#define TR_RCF_LEN_MASK 0x1f00 -#define TR_RCF_DIR 0x0080 -#define TR_RCF_LF_MASK 0x0070 - -#define TR_RCF_RIFLEN(x) ((ntohs(x) & TR_RCF_LEN_MASK) >> 8) - -/* - * Minimum and maximum packet payload lengths. - */ -#define ISO88025_MIN_LEN 0 -#define ISO88025_MAX_LEN_4 4464 -#define ISO88025_MAX_LEN_16 17960 -#define ISO88025_MAX_LEN ISO88025_MAX_LEN_16 - -/* - * A macro to validate a length with - */ -#define ISO88025_IS_VALID_LEN(foo) \ - ((foo) >= ISO88025_MIN_LEN && (foo) <= ISO88025_MAX_LEN) - -/* Access Control field */ -#define AC_PRI_MASK 0xe0 /* Priority bits */ -#define AC_TOKEN 0x10 /* Token bit: 0=Token, 1=Frame */ -#define AC_MONITOR 0x08 /* Monitor */ -#define AC_RESV_MASK 0x07 /* Reservation bits */ - -/* Frame Control field */ -#define FC_FT_MASK 0xc0 /* Frame Type */ -#define FC_FT_MAC 0x00 /* MAC frame */ -#define FC_FT_LLC 0x40 /* LLC frame */ -#define FC_ATTN_MASK 0x0f /* Attention bits */ -#define FC_ATTN_EB 0x01 /* Express buffer */ -#define FC_ATTN_BE 0x02 /* Beacon */ -#define FC_ATTN_CT 0x03 /* Claim token */ -#define FC_ATTN_RP 0x04 /* Ring purge */ -#define FC_ATTN_AMP 0x05 /* Active monitor present */ -#define FC_ATTN_SMP 0x06 /* Standby monitor present */ - -/* Token Ring destination address */ -#define DA_IG 0x80 /* Individual/group address. */ - /* 0=Individual, 1=Group */ -#define DA_UL 0x40 /* Universal/local address. */ - /* 0=Universal, 1=Local */ -/* Token Ring source address */ -#define SA_RII 0x80 /* Routing information indicator */ -#define SA_IG 0x40 /* Individual/group address */ - /* 0=Group, 1=Individual */ - -/* - * ISO 802.5 physical header - */ -struct iso88025_header { - u_int8_t ac; /* access control field */ - u_int8_t fc; /* frame control field */ - u_int8_t iso88025_dhost[ISO88025_ADDR_LEN]; /* destination address */ - u_int8_t iso88025_shost[ISO88025_ADDR_LEN]; /* source address */ - u_int16_t rcf; /* route control field */ - u_int16_t rd[RIF_MAX_RD]; /* routing designators */ -} __packed; - -struct iso88025_rif { - u_int16_t rcf; /* route control field */ - u_int16_t rd[RIF_MAX_RD]; /* routing designators */ -} __packed; - -struct iso88025_sockaddr_data { - u_char ether_dhost[ISO88025_ADDR_LEN]; - u_char ether_shost[ISO88025_ADDR_LEN]; - u_char ac; - u_char fc; -}; - -struct iso88025_sockaddr_dl_data { - u_short trld_rcf; - u_short *trld_route[RIF_MAX_LEN]; -}; - -#define ISO88025_MAX(a, b) (((a)>(b))?(a):(b)) -#define SDL_ISO88025(s) ((struct iso88025_sockaddr_dl_data *) \ - ((s)->sdl_data + \ - ISO88025_MAX((s)->sdl_nlen + (s)->sdl_alen + \ - (s)->sdl_slen, 12))) - -/* - * Structure of a 48-bit iso 802.5 address. - * ( We could also add the 16 bit addresses as a union) - */ -struct iso88025_addr { - u_char octet[ISO88025_ADDR_LEN]; -}; - -#define ISO88025_MAX_MTU 18000 -#define ISO88025_DEFAULT_MTU 1500 - -#define ISO88025_BPF_UNSUPPORTED 0 -#define ISO88025_BPF_SUPPORTED 1 - -#ifdef _KERNEL -void iso88025_ifattach (struct ifnet *, const u_int8_t *, int); -void iso88025_ifdetach (struct ifnet *, int); -int iso88025_ioctl (struct ifnet *, u_long, caddr_t ); -int iso88025_output (struct ifnet *, struct mbuf *, - const struct sockaddr *, struct route *); -void iso88025_input (struct ifnet *, struct mbuf *); -#endif /* _KERNEL */ - -#endif /* !_NET_ISO88025_H_ */ diff --git a/freebsd/sys/net/pfil.c b/freebsd/sys/net/pfil.c index e29cd5e7..65af515f 100644 --- a/freebsd/sys/net/pfil.c +++ b/freebsd/sys/net/pfil.c @@ -59,7 +59,8 @@ MTX_SYSINIT(pfil_heads_lock, &pfil_global_lock, "pfil_head_list lock", static struct packet_filter_hook *pfil_chain_get(int, struct pfil_head *); static int pfil_chain_add(pfil_chain_t *, struct packet_filter_hook *, int); -static int pfil_chain_remove(pfil_chain_t *, pfil_func_t, void *); +static int pfil_chain_remove(pfil_chain_t *, void *, void *); +static int pfil_add_hook_priv(void *, void *, int, struct pfil_head *, bool); LIST_HEAD(pfilheadhead, pfil_head); VNET_DEFINE(struct pfilheadhead, pfil_head_list); @@ -97,7 +98,7 @@ VNET_DEFINE(struct rmlock, pfil_lock); */ int pfil_run_hooks(struct pfil_head *ph, struct mbuf **mp, struct ifnet *ifp, - int dir, struct inpcb *inp) + int dir, int flags, struct inpcb *inp) { struct rm_priotracker rmpt; struct packet_filter_hook *pfh; @@ -108,6 +109,12 @@ pfil_run_hooks(struct pfil_head *ph, struct mbuf **mp, struct ifnet *ifp, KASSERT(ph->ph_nhooks >= 0, ("Pfil hook count dropped < 0")); for (pfh = pfil_chain_get(dir, ph); pfh != NULL; pfh = TAILQ_NEXT(pfh, pfil_chain)) { + if (pfh->pfil_func_flags != NULL) { + rv = (*pfh->pfil_func_flags)(pfh->pfil_arg, &m, ifp, + dir, flags, inp); + if (rv != 0 || m == NULL) + break; + } if (pfh->pfil_func != NULL) { rv = (*pfh->pfil_func)(pfh->pfil_arg, &m, ifp, dir, inp); @@ -260,6 +267,21 @@ pfil_head_get(int type, u_long val) } /* + * pfil_add_hook_flags() adds a function to the packet filter hook. the + * flags are: + * PFIL_IN call me on incoming packets + * PFIL_OUT call me on outgoing packets + * PFIL_ALL call me on all of the above + * PFIL_WAITOK OK to call malloc with M_WAITOK. + */ +int +pfil_add_hook_flags(pfil_func_flags_t func, void *arg, int flags, + struct pfil_head *ph) +{ + return (pfil_add_hook_priv(func, arg, flags, ph, true)); +} + +/* * pfil_add_hook() adds a function to the packet filter hook. the * flags are: * PFIL_IN call me on incoming packets @@ -270,6 +292,13 @@ pfil_head_get(int type, u_long val) int pfil_add_hook(pfil_func_t func, void *arg, int flags, struct pfil_head *ph) { + return (pfil_add_hook_priv(func, arg, flags, ph, false)); +} + +static int +pfil_add_hook_priv(void *func, void *arg, int flags, + struct pfil_head *ph, bool hasflags) +{ struct packet_filter_hook *pfh1 = NULL; struct packet_filter_hook *pfh2 = NULL; int err; @@ -292,7 +321,8 @@ pfil_add_hook(pfil_func_t func, void *arg, int flags, struct pfil_head *ph) } PFIL_WLOCK(ph); if (flags & PFIL_IN) { - pfh1->pfil_func = func; + pfh1->pfil_func_flags = hasflags ? func : NULL; + pfh1->pfil_func = hasflags ? NULL : func; pfh1->pfil_arg = arg; err = pfil_chain_add(&ph->ph_in, pfh1, flags & ~PFIL_OUT); if (err) @@ -300,7 +330,8 @@ pfil_add_hook(pfil_func_t func, void *arg, int flags, struct pfil_head *ph) ph->ph_nhooks++; } if (flags & PFIL_OUT) { - pfh2->pfil_func = func; + pfh2->pfil_func_flags = hasflags ? func : NULL; + pfh2->pfil_func = hasflags ? NULL : func; pfh2->pfil_arg = arg; err = pfil_chain_add(&ph->ph_out, pfh2, flags & ~PFIL_IN); if (err) { @@ -323,6 +354,17 @@ error: } /* + * pfil_remove_hook_flags removes a specific function from the packet filter hook + * chain. + */ +int +pfil_remove_hook_flags(pfil_func_flags_t func, void *arg, int flags, + struct pfil_head *ph) +{ + return (pfil_remove_hook((pfil_func_t)func, arg, flags, ph)); +} + +/* * pfil_remove_hook removes a specific function from the packet filter hook * chain. */ @@ -358,7 +400,9 @@ pfil_chain_add(pfil_chain_t *chain, struct packet_filter_hook *pfh1, int flags) * First make sure the hook is not already there. */ TAILQ_FOREACH(pfh, chain, pfil_chain) - if (pfh->pfil_func == pfh1->pfil_func && + if (((pfh->pfil_func != NULL && pfh->pfil_func == pfh1->pfil_func) || + (pfh->pfil_func_flags != NULL && + pfh->pfil_func_flags == pfh1->pfil_func_flags)) && pfh->pfil_arg == pfh1->pfil_arg) return (EEXIST); @@ -377,12 +421,13 @@ pfil_chain_add(pfil_chain_t *chain, struct packet_filter_hook *pfh1, int flags) * Internal: Remove a pfil hook from a hook chain. */ static int -pfil_chain_remove(pfil_chain_t *chain, pfil_func_t func, void *arg) +pfil_chain_remove(pfil_chain_t *chain, void *func, void *arg) { struct packet_filter_hook *pfh; TAILQ_FOREACH(pfh, chain, pfil_chain) - if (pfh->pfil_func == func && pfh->pfil_arg == arg) { + if ((pfh->pfil_func == func || pfh->pfil_func_flags == func) && + pfh->pfil_arg == arg) { TAILQ_REMOVE(chain, pfh, pfil_chain); free(pfh, M_IFADDR); return (0); diff --git a/freebsd/sys/net/pfil.h b/freebsd/sys/net/pfil.h index 2243ad83..8fdaf5a6 100644 --- a/freebsd/sys/net/pfil.h +++ b/freebsd/sys/net/pfil.h @@ -48,6 +48,8 @@ struct inpcb; typedef int (*pfil_func_t)(void *, struct mbuf **, struct ifnet *, int, struct inpcb *); +typedef int (*pfil_func_flags_t)(void *, struct mbuf **, struct ifnet *, + int, int, struct inpcb *); /* * The packet filter hooks are designed for anything to call them to @@ -56,13 +58,15 @@ typedef int (*pfil_func_t)(void *, struct mbuf **, struct ifnet *, int, */ struct packet_filter_hook { TAILQ_ENTRY(packet_filter_hook) pfil_chain; - pfil_func_t pfil_func; - void *pfil_arg; + pfil_func_t pfil_func; + pfil_func_flags_t pfil_func_flags; + void *pfil_arg; }; #define PFIL_IN 0x00000001 #define PFIL_OUT 0x00000002 #define PFIL_WAITOK 0x00000004 +#define PFIL_FWD 0x00000008 #define PFIL_ALL (PFIL_IN|PFIL_OUT) typedef TAILQ_HEAD(pfil_chain, packet_filter_hook) pfil_chain_t; @@ -102,13 +106,15 @@ VNET_DECLARE(struct rmlock, pfil_lock); /* Public functions for pfil hook management by packet filters. */ struct pfil_head *pfil_head_get(int, u_long); +int pfil_add_hook_flags(pfil_func_flags_t, void *, int, struct pfil_head *); int pfil_add_hook(pfil_func_t, void *, int, struct pfil_head *); +int pfil_remove_hook_flags(pfil_func_flags_t, void *, int, struct pfil_head *); int pfil_remove_hook(pfil_func_t, void *, int, struct pfil_head *); #define PFIL_HOOKED(p) ((p)->ph_nhooks > 0) /* Public functions to run the packet inspection by protocols. */ -int pfil_run_hooks(struct pfil_head *, struct mbuf **, struct ifnet *, - int, struct inpcb *inp); +int pfil_run_hooks(struct pfil_head *, struct mbuf **, struct ifnet *, int, + int, struct inpcb *inp); /* Public functions for pfil head management by protocols. */ int pfil_head_register(struct pfil_head *); diff --git a/freebsd/sys/net/pfvar.h b/freebsd/sys/net/pfvar.h index c9b1d1db..dfff0b87 100644 --- a/freebsd/sys/net/pfvar.h +++ b/freebsd/sys/net/pfvar.h @@ -1467,6 +1467,7 @@ struct pf_idhash { extern u_long pf_hashmask; extern u_long pf_srchashmask; #define PF_HASHSIZ (32768) +#define PF_SRCHASHSIZ (PF_HASHSIZ/4) VNET_DECLARE(struct pf_keyhash *, pf_keyhash); VNET_DECLARE(struct pf_idhash *, pf_idhash); #define V_pf_keyhash VNET(pf_keyhash) @@ -1577,13 +1578,13 @@ extern void pf_addrcpy(struct pf_addr *, struct pf_addr *, void pf_free_rule(struct pf_rule *); #ifdef INET -int pf_test(int, struct ifnet *, struct mbuf **, struct inpcb *); +int pf_test(int, int, struct ifnet *, struct mbuf **, struct inpcb *); int pf_normalize_ip(struct mbuf **, int, struct pfi_kif *, u_short *, struct pf_pdesc *); #endif /* INET */ #ifdef INET6 -int pf_test6(int, struct ifnet *, struct mbuf **, struct inpcb *); +int pf_test6(int, int, struct ifnet *, struct mbuf **, struct inpcb *); int pf_normalize_ip6(struct mbuf **, int, struct pfi_kif *, u_short *, struct pf_pdesc *); void pf_poolmask(struct pf_addr *, struct pf_addr*, diff --git a/freebsd/sys/net/rtsock.c b/freebsd/sys/net/rtsock.c index 851c30c9..900413a0 100644 --- a/freebsd/sys/net/rtsock.c +++ b/freebsd/sys/net/rtsock.c @@ -686,12 +686,15 @@ route_output(struct mbuf *m, struct socket *so, ...) case RTM_ADD: case RTM_CHANGE: - if (info.rti_info[RTAX_GATEWAY] == NULL) - senderr(EINVAL); + if (rtm->rtm_type == RTM_ADD) { + if (info.rti_info[RTAX_GATEWAY] == NULL) + senderr(EINVAL); + } saved_nrt = NULL; /* support for new ARP code */ - if (info.rti_info[RTAX_GATEWAY]->sa_family == AF_LINK && + if (info.rti_info[RTAX_GATEWAY] != NULL && + info.rti_info[RTAX_GATEWAY]->sa_family == AF_LINK && (rtm->rtm_flags & RTF_LLDATA) != 0) { error = lla_rt_output(rtm, &info); #ifdef INET6 diff --git a/freebsd/sys/net80211/ieee80211_ioctl.c b/freebsd/sys/net80211/ieee80211_ioctl.c index c5f6721d..52712514 100644 --- a/freebsd/sys/net80211/ieee80211_ioctl.c +++ b/freebsd/sys/net80211/ieee80211_ioctl.c @@ -3569,7 +3569,8 @@ ieee80211_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) break; case SIOCG80211STATS: ifr = (struct ifreq *)data; - copyout(&vap->iv_stats, ifr->ifr_data, sizeof (vap->iv_stats)); + copyout(&vap->iv_stats, ifr_data_get_ptr(ifr), + sizeof (vap->iv_stats)); break; case SIOCSIFMTU: ifr = (struct ifreq *)data; diff --git a/freebsd/sys/net80211/ieee80211_output.c b/freebsd/sys/net80211/ieee80211_output.c index a49c8a16..06fca965 100644 --- a/freebsd/sys/net80211/ieee80211_output.c +++ b/freebsd/sys/net80211/ieee80211_output.c @@ -3090,6 +3090,39 @@ ieee80211_alloc_cts(struct ieee80211com *ic, return m; } +/* + * Wrapper for CTS/RTS frame allocation. + */ +struct mbuf * +ieee80211_alloc_prot(struct ieee80211_node *ni, const struct mbuf *m, + uint8_t rate, int prot) +{ + struct ieee80211com *ic = ni->ni_ic; + const struct ieee80211_frame *wh; + struct mbuf *mprot; + uint16_t dur; + int pktlen, isshort; + + KASSERT(prot == IEEE80211_PROT_RTSCTS || + prot == IEEE80211_PROT_CTSONLY, + ("wrong protection type %d", prot)); + + wh = mtod(m, const struct ieee80211_frame *); + pktlen = m->m_pkthdr.len + IEEE80211_CRC_LEN; + isshort = (ic->ic_flags & IEEE80211_F_SHPREAMBLE) != 0; + dur = ieee80211_compute_duration(ic->ic_rt, pktlen, rate, isshort) + + ieee80211_ack_duration(ic->ic_rt, rate, isshort); + + if (prot == IEEE80211_PROT_RTSCTS) { + /* NB: CTS is the same size as an ACK */ + dur += ieee80211_ack_duration(ic->ic_rt, rate, isshort); + mprot = ieee80211_alloc_rts(ic, wh->i_addr1, wh->i_addr2, dur); + } else + mprot = ieee80211_alloc_cts(ic, ni->ni_vap->iv_myaddr, dur); + + return (mprot); +} + static void ieee80211_tx_mgt_timeout(void *arg) { diff --git a/freebsd/sys/net80211/ieee80211_proto.h b/freebsd/sys/net80211/ieee80211_proto.h index 28238ce8..c1637c57 100644 --- a/freebsd/sys/net80211/ieee80211_proto.h +++ b/freebsd/sys/net80211/ieee80211_proto.h @@ -140,6 +140,8 @@ struct mbuf *ieee80211_alloc_rts(struct ieee80211com *ic, const uint8_t [IEEE80211_ADDR_LEN], uint16_t); struct mbuf *ieee80211_alloc_cts(struct ieee80211com *, const uint8_t [IEEE80211_ADDR_LEN], uint16_t); +struct mbuf *ieee80211_alloc_prot(struct ieee80211_node *, + const struct mbuf *, uint8_t, int); uint8_t *ieee80211_add_rates(uint8_t *, const struct ieee80211_rateset *); uint8_t *ieee80211_add_xrates(uint8_t *, const struct ieee80211_rateset *); diff --git a/freebsd/sys/netinet/cc/cc.c b/freebsd/sys/netinet/cc/cc.c index dc919172..43065ec6 100644 --- a/freebsd/sys/netinet/cc/cc.c +++ b/freebsd/sys/netinet/cc/cc.c @@ -333,3 +333,14 @@ SYSCTL_PROC(_net_inet_tcp_cc, OID_AUTO, available, CTLTYPE_STRING|CTLFLAG_RD, NULL, 0, cc_list_available, "A", "List available congestion control algorithms"); #endif /* __rtems__ */ + +VNET_DEFINE(int, cc_do_abe) = 0; +SYSCTL_INT(_net_inet_tcp_cc, OID_AUTO, abe, CTLFLAG_VNET | CTLFLAG_RW, + &VNET_NAME(cc_do_abe), 0, + "Enable draft-ietf-tcpm-alternativebackoff-ecn (TCP Alternative Backoff with ECN)"); + +VNET_DEFINE(int, cc_abe_frlossreduce) = 0; +SYSCTL_INT(_net_inet_tcp_cc, OID_AUTO, abe_frlossreduce, CTLFLAG_VNET | CTLFLAG_RW, + &VNET_NAME(cc_abe_frlossreduce), 0, + "Apply standard beta instead of ABE-beta during ECN-signalled congestion " + "recovery episodes if loss also needs to be repaired"); diff --git a/freebsd/sys/netinet/cc/cc.h b/freebsd/sys/netinet/cc/cc.h index 9b6279de..1389b27a 100644 --- a/freebsd/sys/netinet/cc/cc.h +++ b/freebsd/sys/netinet/cc/cc.h @@ -64,6 +64,12 @@ extern struct cc_algo newreno_cc_algo; VNET_DECLARE(struct cc_algo *, default_cc_ptr); #define V_default_cc_ptr VNET(default_cc_ptr) +VNET_DECLARE(int, cc_do_abe); +#define V_cc_do_abe VNET(cc_do_abe) + +VNET_DECLARE(int, cc_abe_frlossreduce); +#define V_cc_abe_frlossreduce VNET(cc_abe_frlossreduce) + /* Define the new net.inet.tcp.cc sysctl tree. */ SYSCTL_DECL(_net_inet_tcp_cc); @@ -96,6 +102,8 @@ struct cc_var { #define CCF_ACKNOW 0x0008 /* Will this ack be sent now? */ #define CCF_IPHDR_CE 0x0010 /* Does this packet set CE bit? */ #define CCF_TCPHDR_CWR 0x0020 /* Does this packet set CWR bit? */ +#define CCF_MAX_CWND 0x0040 /* Have we reached maximum cwnd? */ +#define CCF_CHG_MAX_CWND 0x0080 /* Cubic max_cwnd changed, for K */ /* ACK types passed to the ack_received() hook. */ #define CC_ACK 0x0001 /* Regular in sequence ACK. */ diff --git a/freebsd/sys/netinet/cc/cc_newreno.c b/freebsd/sys/netinet/cc/cc_newreno.c index 58a6504a..b7f59520 100644 --- a/freebsd/sys/netinet/cc/cc_newreno.c +++ b/freebsd/sys/netinet/cc/cc_newreno.c @@ -5,7 +5,7 @@ * * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1994, 1995 * The Regents of the University of California. - * Copyright (c) 2007-2008,2010 + * Copyright (c) 2007-2008,2010,2014 * Swinburne University of Technology, Melbourne, Australia. * Copyright (c) 2009-2010 Lawrence Stewart <lstewart@freebsd.org> * Copyright (c) 2010 The FreeBSD Foundation @@ -50,6 +50,11 @@ * University Research Program Fund at Community Foundation Silicon Valley. * More details are available at: * http://caia.swin.edu.au/urp/newtcp/ + * + * Dec 2014 garmitage@swin.edu.au + * Borrowed code fragments from cc_cdg.c to add modifiable beta + * via sysctls. + * */ #include <sys/cdefs.h> @@ -71,20 +76,54 @@ __FBSDID("$FreeBSD$"); #include <netinet/tcp_var.h> #include <netinet/cc/cc.h> #include <netinet/cc/cc_module.h> +#include <netinet/cc/cc_newreno.h> + +static MALLOC_DEFINE(M_NEWRENO, "newreno data", + "newreno beta values"); + +#define CAST_PTR_INT(X) (*((int*)(X))) +static int newreno_cb_init(struct cc_var *ccv); static void newreno_ack_received(struct cc_var *ccv, uint16_t type); static void newreno_after_idle(struct cc_var *ccv); static void newreno_cong_signal(struct cc_var *ccv, uint32_t type); static void newreno_post_recovery(struct cc_var *ccv); +static int newreno_ctl_output(struct cc_var *ccv, struct sockopt *sopt, void *buf); + +static VNET_DEFINE(uint32_t, newreno_beta) = 50; +static VNET_DEFINE(uint32_t, newreno_beta_ecn) = 80; +#define V_newreno_beta VNET(newreno_beta) +#define V_newreno_beta_ecn VNET(newreno_beta_ecn) struct cc_algo newreno_cc_algo = { .name = "newreno", + .cb_init = newreno_cb_init, .ack_received = newreno_ack_received, .after_idle = newreno_after_idle, .cong_signal = newreno_cong_signal, .post_recovery = newreno_post_recovery, + .ctl_output = newreno_ctl_output, +}; + +struct newreno { + uint32_t beta; + uint32_t beta_ecn; }; +int +newreno_cb_init(struct cc_var *ccv) +{ + struct newreno *nreno; + + nreno = malloc(sizeof(struct newreno), M_NEWRENO, M_NOWAIT|M_ZERO); + if (nreno != NULL) { + nreno->beta = V_newreno_beta; + nreno->beta_ecn = V_newreno_beta_ecn; + } + + return (0); +} + static void newreno_ack_received(struct cc_var *ccv, uint16_t type) { @@ -186,27 +225,48 @@ newreno_after_idle(struct cc_var *ccv) static void newreno_cong_signal(struct cc_var *ccv, uint32_t type) { - u_int win; + struct newreno *nreno; + uint32_t cwin, factor; + u_int mss; + + factor = V_newreno_beta; + nreno = ccv->cc_data; + if (nreno != NULL) { + if (V_cc_do_abe) + factor = (type == CC_ECN ? nreno->beta_ecn: nreno->beta); + else + factor = nreno->beta; + } + + cwin = CCV(ccv, snd_cwnd); + mss = CCV(ccv, t_maxseg); /* Catch algos which mistakenly leak private signal types. */ KASSERT((type & CC_SIGPRIVMASK) == 0, ("%s: congestion signal type 0x%08x is private\n", __func__, type)); - win = max(CCV(ccv, snd_cwnd) / 2 / CCV(ccv, t_maxseg), 2) * - CCV(ccv, t_maxseg); + cwin = max(((uint64_t)cwin * (uint64_t)factor) / (100ULL * (uint64_t)mss), + 2) * mss; switch (type) { case CC_NDUPACK: if (!IN_FASTRECOVERY(CCV(ccv, t_flags))) { + if (IN_CONGRECOVERY(CCV(ccv, t_flags) && + V_cc_do_abe && V_cc_abe_frlossreduce)) { + CCV(ccv, snd_ssthresh) = + ((uint64_t)CCV(ccv, snd_ssthresh) * + (uint64_t)nreno->beta) / + (100ULL * (uint64_t)nreno->beta_ecn); + } if (!IN_CONGRECOVERY(CCV(ccv, t_flags))) - CCV(ccv, snd_ssthresh) = win; + CCV(ccv, snd_ssthresh) = cwin; ENTER_RECOVERY(CCV(ccv, t_flags)); } break; case CC_ECN: if (!IN_CONGRECOVERY(CCV(ccv, t_flags))) { - CCV(ccv, snd_ssthresh) = win; - CCV(ccv, snd_cwnd) = win; + CCV(ccv, snd_ssthresh) = cwin; + CCV(ccv, snd_cwnd) = cwin; ENTER_CONGRECOVERY(CCV(ccv, t_flags)); } break; @@ -244,5 +304,75 @@ newreno_post_recovery(struct cc_var *ccv) } } +int +newreno_ctl_output(struct cc_var *ccv, struct sockopt *sopt, void *buf) +{ + struct newreno *nreno; + struct cc_newreno_opts *opt; + + if (sopt->sopt_valsize != sizeof(struct cc_newreno_opts)) + return (EMSGSIZE); + + nreno = ccv->cc_data; + opt = buf; + + switch (sopt->sopt_dir) { + case SOPT_SET: + switch (opt->name) { + case CC_NEWRENO_BETA: + nreno->beta = opt->val; + break; + case CC_NEWRENO_BETA_ECN: + if (!V_cc_do_abe) + return (EACCES); + nreno->beta_ecn = opt->val; + break; + default: + return (ENOPROTOOPT); + } + case SOPT_GET: + switch (opt->name) { + case CC_NEWRENO_BETA: + opt->val = nreno->beta; + break; + case CC_NEWRENO_BETA_ECN: + opt->val = nreno->beta_ecn; + break; + default: + return (ENOPROTOOPT); + } + default: + return (EINVAL); + } + + return (0); +} + +static int +newreno_beta_handler(SYSCTL_HANDLER_ARGS) +{ + if (req->newptr != NULL ) { + if (arg1 == &VNET_NAME(newreno_beta_ecn) && !V_cc_do_abe) + return (EACCES); + if (CAST_PTR_INT(req->newptr) <= 0 || CAST_PTR_INT(req->newptr) > 100) + return (EINVAL); + } + + return (sysctl_handle_int(oidp, arg1, arg2, req)); +} + +SYSCTL_DECL(_net_inet_tcp_cc_newreno); +SYSCTL_NODE(_net_inet_tcp_cc, OID_AUTO, newreno, CTLFLAG_RW, NULL, + "New Reno related settings"); + +SYSCTL_PROC(_net_inet_tcp_cc_newreno, OID_AUTO, beta, + CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, + &VNET_NAME(newreno_beta), 3, &newreno_beta_handler, "IU", + "New Reno beta, specified as number between 1 and 100"); + +SYSCTL_PROC(_net_inet_tcp_cc_newreno, OID_AUTO, beta_ecn, + CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, + &VNET_NAME(newreno_beta_ecn), 3, &newreno_beta_handler, "IU", + "New Reno beta ecn, specified as number between 1 and 100"); DECLARE_CC_MODULE(newreno, &newreno_cc_algo); diff --git a/freebsd/sys/netinet/cc/cc_newreno.h b/freebsd/sys/netinet/cc/cc_newreno.h new file mode 100644 index 00000000..9e2a3cff --- /dev/null +++ b/freebsd/sys/netinet/cc/cc_newreno.h @@ -0,0 +1,42 @@ +/*- + * Copyright (c) 2017 Tom Jones <tj@enoti.me> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _CC_NEWRENO_H +#define _CC_NEWRENO_H + +#define CCALGONAME_NEWRENO "newreno" + +struct cc_newreno_opts { + int name; + uint32_t val; +}; + +#define CC_NEWRENO_BETA 1 +#define CC_NEWRENO_BETA_ECN 2 + +#endif /* _CC_NEWRENO_H */ diff --git a/freebsd/sys/netinet/if_ether.c b/freebsd/sys/netinet/if_ether.c index fbf6238f..699af2e4 100644 --- a/freebsd/sys/netinet/if_ether.c +++ b/freebsd/sys/netinet/if_ether.c @@ -506,12 +506,8 @@ arpresolve_full(struct ifnet *ifp, int is_gw, int flags, struct mbuf *m, } bcopy(lladdr, desten, ll_len); - /* Check if we have feedback request from arptimer() */ - if (la->r_skip_req != 0) { - LLE_REQ_LOCK(la); - la->r_skip_req = 0; /* Notify that entry was used */ - LLE_REQ_UNLOCK(la); - } + /* Notify LLE code that the entry was used by datapath */ + llentry_mark_used(la); if (pflags != NULL) *pflags = la->la_flags & (LLE_VALID|LLE_IFADDR); if (plle) { @@ -642,12 +638,8 @@ arpresolve(struct ifnet *ifp, int is_gw, struct mbuf *m, bcopy(la->r_linkdata, desten, la->r_hdrlen); if (pflags != NULL) *pflags = LLE_VALID | (la->r_flags & RLLE_IFADDR); - /* Check if we have feedback request from arptimer() */ - if (la->r_skip_req != 0) { - LLE_REQ_LOCK(la); - la->r_skip_req = 0; /* Notify that entry was used */ - LLE_REQ_UNLOCK(la); - } + /* Notify the LLE handling code that the entry was used. */ + llentry_mark_used(la); if (plle) { LLE_ADDREF(la); *plle = la; diff --git a/freebsd/sys/netinet/in.c b/freebsd/sys/netinet/in.c index 77ada362..28c257aa 100644 --- a/freebsd/sys/netinet/in.c +++ b/freebsd/sys/netinet/in.c @@ -1072,6 +1072,19 @@ in_lltable_destroy_lle_unlocked(struct llentry *lle) } /* + * Called by the datapath to indicate that + * the entry was used. + */ +static void +in_lltable_mark_used(struct llentry *lle) +{ + + LLE_REQ_LOCK(lle); + lle->r_skip_req = 0; + LLE_REQ_UNLOCK(lle); +} + +/* * Called by LLE_FREE_LOCKED when number of references * drops to zero. */ @@ -1474,6 +1487,7 @@ in_lltattach(struct ifnet *ifp) llt->llt_fill_sa_entry = in_lltable_fill_sa_entry; llt->llt_free_entry = in_lltable_free_entry; llt->llt_match_prefix = in_lltable_match_prefix; + llt->llt_mark_used = in_lltable_mark_used; lltable_link(llt); return (llt); diff --git a/freebsd/sys/netinet/in_pcb.c b/freebsd/sys/netinet/in_pcb.c index 2803126d..0d388132 100644 --- a/freebsd/sys/netinet/in_pcb.c +++ b/freebsd/sys/netinet/in_pcb.c @@ -1644,6 +1644,7 @@ in_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup, struct inpcbhead *head; struct inpcb *inp, *tmpinp; u_short fport = fport_arg, lport = lport_arg; + bool locked; /* * First look for an exact match. @@ -1830,18 +1831,32 @@ in_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup, return (NULL); found: - in_pcbref(inp); - INP_GROUP_UNLOCK(pcbgroup); - if (lookupflags & INPLOOKUP_WLOCKPCB) { - INP_WLOCK(inp); - if (in_pcbrele_wlocked(inp)) - return (NULL); - } else if (lookupflags & INPLOOKUP_RLOCKPCB) { - INP_RLOCK(inp); - if (in_pcbrele_rlocked(inp)) - return (NULL); - } else + if (lookupflags & INPLOOKUP_WLOCKPCB) + locked = INP_TRY_WLOCK(inp); + else if (lookupflags & INPLOOKUP_RLOCKPCB) + locked = INP_TRY_RLOCK(inp); + else panic("%s: locking bug", __func__); + if (!locked) + in_pcbref(inp); + INP_GROUP_UNLOCK(pcbgroup); + if (!locked) { + if (lookupflags & INPLOOKUP_WLOCKPCB) { + INP_WLOCK(inp); + if (in_pcbrele_wlocked(inp)) + return (NULL); + } else { + INP_RLOCK(inp); + if (in_pcbrele_rlocked(inp)) + return (NULL); + } + } +#ifdef INVARIANTS + if (lookupflags & INPLOOKUP_WLOCKPCB) + INP_WLOCK_ASSERT(inp); + else + INP_RLOCK_ASSERT(inp); +#endif return (inp); } #endif /* PCBGROUP */ @@ -1980,23 +1995,38 @@ in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr, struct ifnet *ifp) { struct inpcb *inp; + bool locked; INP_HASH_RLOCK(pcbinfo); inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport, (lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp); if (inp != NULL) { - in_pcbref(inp); - INP_HASH_RUNLOCK(pcbinfo); - if (lookupflags & INPLOOKUP_WLOCKPCB) { - INP_WLOCK(inp); - if (in_pcbrele_wlocked(inp)) - return (NULL); - } else if (lookupflags & INPLOOKUP_RLOCKPCB) { - INP_RLOCK(inp); - if (in_pcbrele_rlocked(inp)) - return (NULL); - } else + if (lookupflags & INPLOOKUP_WLOCKPCB) + locked = INP_TRY_WLOCK(inp); + else if (lookupflags & INPLOOKUP_RLOCKPCB) + locked = INP_TRY_RLOCK(inp); + else panic("%s: locking bug", __func__); + if (!locked) + in_pcbref(inp); + INP_HASH_RUNLOCK(pcbinfo); + if (!locked) { + if (lookupflags & INPLOOKUP_WLOCKPCB) { + INP_WLOCK(inp); + if (in_pcbrele_wlocked(inp)) + return (NULL); + } else { + INP_RLOCK(inp); + if (in_pcbrele_rlocked(inp)) + return (NULL); + } + } +#ifdef INVARIANTS + if (lookupflags & INPLOOKUP_WLOCKPCB) + INP_WLOCK_ASSERT(inp); + else + INP_RLOCK_ASSERT(inp); +#endif } else INP_HASH_RUNLOCK(pcbinfo); return (inp); diff --git a/freebsd/sys/netinet/ip_carp.c b/freebsd/sys/netinet/ip_carp.c index 70537cfb..e2bd0a0a 100644 --- a/freebsd/sys/netinet/ip_carp.c +++ b/freebsd/sys/netinet/ip_carp.c @@ -63,7 +63,6 @@ __FBSDID("$FreeBSD$"); #include <net/if_dl.h> #include <net/if_llatbl.h> #include <net/if_types.h> -#include <net/iso88025.h> #include <net/route.h> #include <net/vnet.h> @@ -1541,17 +1540,6 @@ carp_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *sa) fh->fddi_shost[5] = sc->sc_vhid; } break; - case IFT_ISO88025: { - struct iso88025_header *th; - th = mtod(m, struct iso88025_header *); - th->iso88025_shost[0] = 3; - th->iso88025_shost[1] = 0; - th->iso88025_shost[2] = 0x40 >> (sc->sc_vhid - 1); - th->iso88025_shost[3] = 0x40000 >> (sc->sc_vhid - 1); - th->iso88025_shost[4] = 0; - th->iso88025_shost[5] = 0; - } - break; default: printf("%s: carp is not supported for the %d interface type\n", ifp->if_xname, ifp->if_type); @@ -1722,7 +1710,7 @@ carp_ioctl(struct ifreq *ifr, u_long cmd, struct thread *td) struct carp_softc *sc = NULL; int error = 0, locked = 0; - if ((error = copyin(ifr->ifr_data, &carpr, sizeof carpr))) + if ((error = copyin(ifr_data_get_ptr(ifr), &carpr, sizeof carpr))) return (error); ifp = ifunit_ref(ifr->ifr_name); @@ -1734,7 +1722,6 @@ carp_ioctl(struct ifreq *ifr, u_long cmd, struct thread *td) case IFT_L2VLAN: case IFT_BRIDGE: case IFT_FDDI: - case IFT_ISO88025: break; default: error = EOPNOTSUPP; @@ -1839,7 +1826,8 @@ carp_ioctl(struct ifreq *ifr, u_long cmd, struct thread *td) break; } carp_carprcp(&carpr, sc, priveleged); - error = copyout(&carpr, ifr->ifr_data, sizeof(carpr)); + error = copyout(&carpr, ifr_data_get_ptr(ifr), + sizeof(carpr)); } else { int i, count; @@ -1857,7 +1845,8 @@ carp_ioctl(struct ifreq *ifr, u_long cmd, struct thread *td) IFNET_FOREACH_CARP(ifp, sc) { carp_carprcp(&carpr, sc, priveleged); carpr.carpr_count = count; - error = copyout(&carpr, ifr->ifr_data + + error = copyout(&carpr, + (caddr_t)ifr_data_get_ptr(ifr) + (i * sizeof(carpr)), sizeof(carpr)); if (error) { CIF_UNLOCK(ifp->if_carp); diff --git a/freebsd/sys/netinet/ip_fastfwd.c b/freebsd/sys/netinet/ip_fastfwd.c index b8a56c31..b084fdc6 100644 --- a/freebsd/sys/netinet/ip_fastfwd.c +++ b/freebsd/sys/netinet/ip_fastfwd.c @@ -234,7 +234,7 @@ ip_tryforward(struct mbuf *m) goto passin; if (pfil_run_hooks( - &V_inet_pfil_hook, &m, m->m_pkthdr.rcvif, PFIL_IN, NULL) || + &V_inet_pfil_hook, &m, m->m_pkthdr.rcvif, PFIL_IN, 0, NULL) || m == NULL) goto drop; @@ -307,8 +307,8 @@ passin: if (!PFIL_HOOKED(&V_inet_pfil_hook)) goto passout; - if (pfil_run_hooks(&V_inet_pfil_hook, &m, nh.nh_ifp, PFIL_OUT, NULL) || - m == NULL) { + if (pfil_run_hooks(&V_inet_pfil_hook, &m, nh.nh_ifp, PFIL_OUT, PFIL_FWD, + NULL) || m == NULL) { goto drop; } diff --git a/freebsd/sys/netinet/ip_fw.h b/freebsd/sys/netinet/ip_fw.h index de938c75..286eb03f 100644 --- a/freebsd/sys/netinet/ip_fw.h +++ b/freebsd/sys/netinet/ip_fw.h @@ -671,7 +671,7 @@ struct ipfw_flow_id { uint32_t src_ip; uint16_t dst_port; uint16_t src_port; - uint8_t fib; + uint8_t fib; /* XXX: must be uint16_t */ uint8_t proto; uint8_t _flags; /* protocol-specific flags */ uint8_t addr_type; /* 4=ip4, 6=ip6, 1=ether ? */ @@ -682,6 +682,7 @@ struct ipfw_flow_id { }; #endif +#define IS_IP4_FLOW_ID(id) ((id)->addr_type == 4) #define IS_IP6_FLOW_ID(id) ((id)->addr_type == 6) /* diff --git a/freebsd/sys/netinet/ip_input.c b/freebsd/sys/netinet/ip_input.c index c1baebbb..2c8bf427 100644 --- a/freebsd/sys/netinet/ip_input.c +++ b/freebsd/sys/netinet/ip_input.c @@ -602,7 +602,7 @@ tooshort: goto passin; odst = ip->ip_dst; - if (pfil_run_hooks(&V_inet_pfil_hook, &m, ifp, PFIL_IN, NULL) != 0) + if (pfil_run_hooks(&V_inet_pfil_hook, &m, ifp, PFIL_IN, 0, NULL) != 0) return; if (m == NULL) /* consumed by filter */ return; diff --git a/freebsd/sys/netinet/ip_output.c b/freebsd/sys/netinet/ip_output.c index 4944c035..21b3919a 100644 --- a/freebsd/sys/netinet/ip_output.c +++ b/freebsd/sys/netinet/ip_output.c @@ -119,7 +119,7 @@ ip_output_pfil(struct mbuf **mp, struct ifnet *ifp, struct inpcb *inp, /* Run through list of hooks for output packets. */ odst.s_addr = ip->ip_dst.s_addr; - *error = pfil_run_hooks(&V_inet_pfil_hook, mp, ifp, PFIL_OUT, inp); + *error = pfil_run_hooks(&V_inet_pfil_hook, mp, ifp, PFIL_OUT, 0, inp); m = *mp; if ((*error) != 0 || m == NULL) return 1; /* Finished */ diff --git a/freebsd/sys/netinet/tcp_fastopen.h b/freebsd/sys/netinet/tcp_fastopen.h new file mode 100644 index 00000000..acb94bb3 --- /dev/null +++ b/freebsd/sys/netinet/tcp_fastopen.h @@ -0,0 +1,108 @@ +/*- + * Copyright (c) 2015-2017 Patrick Kelsey + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _TCP_FASTOPEN_H_ +#define _TCP_FASTOPEN_H_ + +#ifdef _KERNEL + +#include <rtems/bsd/local/opt_inet.h> + +#define TCP_FASTOPEN_COOKIE_LEN 8 /* SipHash24 64-bit output */ + +#ifdef TCP_RFC7413 +VNET_DECLARE(unsigned int, tcp_fastopen_client_enable); +#define V_tcp_fastopen_client_enable VNET(tcp_fastopen_client_enable) + +VNET_DECLARE(unsigned int, tcp_fastopen_server_enable); +#define V_tcp_fastopen_server_enable VNET(tcp_fastopen_server_enable) +#else +#define V_tcp_fastopen_client_enable 0 +#define V_tcp_fastopen_server_enable 0 +#endif /* TCP_RFC7413 */ + +union tcp_fastopen_ip_addr { + struct in_addr v4; + struct in6_addr v6; +}; + +struct tcp_fastopen_ccache_entry { + TAILQ_ENTRY(tcp_fastopen_ccache_entry) cce_link; + union tcp_fastopen_ip_addr cce_client_ip; /* network byte order */ + union tcp_fastopen_ip_addr cce_server_ip; /* network byte order */ + uint16_t server_port; /* network byte order */ + uint16_t server_mss; /* host byte order */ + uint8_t af; + uint8_t cookie_len; + uint8_t cookie[TCP_FASTOPEN_MAX_COOKIE_LEN]; + sbintime_t disable_time; /* non-zero value means path is disabled */ +}; + +struct tcp_fastopen_ccache; + +struct tcp_fastopen_ccache_bucket { + struct mtx ccb_mtx; + TAILQ_HEAD(bucket_entries, tcp_fastopen_ccache_entry) ccb_entries; + int ccb_num_entries; + struct tcp_fastopen_ccache *ccb_ccache; +}; + +struct tcp_fastopen_ccache { + uma_zone_t zone; + struct tcp_fastopen_ccache_bucket *base; + unsigned int bucket_limit; + unsigned int buckets; + unsigned int mask; + uint32_t secret; +}; + +#ifdef TCP_RFC7413 +void tcp_fastopen_init(void); +void tcp_fastopen_destroy(void); +unsigned int *tcp_fastopen_alloc_counter(void); +void tcp_fastopen_decrement_counter(unsigned int *); +int tcp_fastopen_check_cookie(struct in_conninfo *, uint8_t *, unsigned int, + uint64_t *); +void tcp_fastopen_connect(struct tcpcb *); +void tcp_fastopen_disable_path(struct tcpcb *); +void tcp_fastopen_update_cache(struct tcpcb *, uint16_t, uint8_t, + uint8_t *); +#else +#define tcp_fastopen_init() ((void)0) +#define tcp_fastopen_destroy() ((void)0) +#define tcp_fastopen_alloc_counter() NULL +#define tcp_fastopen_decrement_counter(c) ((void)0) +#define tcp_fastopen_check_cookie(i, c, l, lc) (-1) +#define tcp_fastopen_connect(t) ((void)0) +#define tcp_fastopen_disable_path(t) ((void)0) +#define tcp_fastopen_update_cache(t, m, l, c) ((void)0) +#endif /* TCP_RFC7413 */ + +#endif /* _KERNEL */ + +#endif /* _TCP_FASTOPEN_H_ */ diff --git a/freebsd/sys/netinet/tcp_input.c b/freebsd/sys/netinet/tcp_input.c index b172205d..7c907da9 100644 --- a/freebsd/sys/netinet/tcp_input.c +++ b/freebsd/sys/netinet/tcp_input.c @@ -102,17 +102,16 @@ __FBSDID("$FreeBSD$"); #include <netinet6/in6_var.h> #include <netinet6/ip6_var.h> #include <netinet6/nd6.h> -#ifdef TCP_RFC7413 -#include <netinet/tcp_fastopen.h> -#endif #include <netinet/tcp.h> #include <netinet/tcp_fsm.h> +#include <netinet/tcp_log_buf.h> #include <netinet/tcp_seq.h> #include <netinet/tcp_timer.h> #include <netinet/tcp_var.h> #include <netinet6/tcp6_var.h> #include <netinet/tcpip.h> #include <netinet/cc/cc.h> +#include <netinet/tcp_fastopen.h> #ifdef TCPPCAP #include <netinet/tcp_pcap.h> #endif @@ -709,6 +708,7 @@ tcp_input(struct mbuf **mp, int *offp, int proto) ip->ip_tos = iptos; /* Re-initialization for later version check */ ip->ip_v = IPVERSION; + ip->ip_hl = off0 >> 2; } if (th->th_sum) { @@ -1131,9 +1131,7 @@ relocked: rstreason = BANDLIM_RST_OPENPORT; goto dropwithreset; } -#ifdef TCP_RFC7413 tfo_socket_result: -#endif if (so == NULL) { /* * We completed the 3-way handshake @@ -1376,12 +1374,9 @@ tfo_socket_result: #endif TCP_PROBE3(debug__input, tp, th, m); tcp_dooptions(&to, optp, optlen, TO_SYN); -#ifdef TCP_RFC7413 if (syncache_add(&inc, &to, th, inp, &so, m, NULL, NULL)) goto tfo_socket_result; -#else - syncache_add(&inc, &to, th, inp, &so, m, NULL, NULL); -#endif + /* * Entry added to syncache and mbuf consumed. * Only the listen socket is unlocked by syncache_add(). @@ -1551,9 +1546,7 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, struct in_conninfo *inc; struct mbuf *mfree; struct tcpopt to; -#ifdef TCP_RFC7413 int tfo_syn; -#endif #ifdef TCPDEBUG /* @@ -1602,6 +1595,8 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, /* Save segment, if requested. */ tcp_pcap_add(th, m, &(tp->t_inpkts)); #endif + TCP_LOG_EVENT(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_IN, 0, + tlen, NULL, true); if ((thflags & TH_SYN) && (thflags & TH_FIN) && V_drop_synfin) { if ((s = tcp_log_addrs(inc, th, NULL, NULL))) { @@ -1718,6 +1713,13 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, if ((tp->t_flags & TF_SACK_PERMIT) && (to.to_flags & TOF_SACKPERM) == 0) tp->t_flags &= ~TF_SACK_PERMIT; + if (IS_FASTOPEN(tp->t_flags)) { + if (to.to_flags & TOF_FASTOPEN) + tcp_fastopen_update_cache(tp, to.to_mss, + to.to_tfo_len, to.to_tfo_cookie); + else + tcp_fastopen_disable_path(tp); + } } /* @@ -1975,7 +1977,6 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, rstreason = BANDLIM_RST_OPENPORT; goto dropwithreset; } -#ifdef TCP_RFC7413 if (IS_FASTOPEN(tp->t_flags)) { /* * When a TFO connection is in SYN_RECEIVED, the @@ -1996,7 +1997,6 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, goto drop; } } -#endif break; /* @@ -2028,6 +2028,8 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, tp->irs = th->th_seq; tcp_rcvseqinit(tp); if (thflags & TH_ACK) { + int tfo_partial_ack = 0; + TCPSTAT_INC(tcps_connects); soisconnected(so); #ifdef MAC @@ -2042,10 +2044,19 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, TCP_MAXWIN << tp->rcv_scale); tp->snd_una++; /* SYN is acked */ /* + * If not all the data that was sent in the TFO SYN + * has been acked, resend the remainder right away. + */ + if (IS_FASTOPEN(tp->t_flags) && + (tp->snd_una != tp->snd_max)) { + tp->snd_nxt = th->th_ack; + tfo_partial_ack = 1; + } + /* * If there's data, delay ACK; if there's also a FIN * ACKNOW will be turned on later. */ - if (DELAY_ACK(tp, tlen) && tlen != 0) + if (DELAY_ACK(tp, tlen) && tlen != 0 && !tfo_partial_ack) tcp_timer_activate(tp, TT_DELACK, tcp_delacktime); else @@ -2404,13 +2415,11 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, if ((thflags & TH_ACK) == 0) { if (tp->t_state == TCPS_SYN_RECEIVED || (tp->t_flags & TF_NEEDSYN)) { -#ifdef TCP_RFC7413 if (tp->t_state == TCPS_SYN_RECEIVED && IS_FASTOPEN(tp->t_flags)) { tp->snd_wnd = tiwin; cc_conn_init(tp); } -#endif goto step6; } else if (tp->t_flags & TF_ACKNOW) goto dropafterack; @@ -2451,8 +2460,7 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, tcp_state_change(tp, TCPS_ESTABLISHED); TCP_PROBE5(accept__established, NULL, tp, m, tp, th); -#ifdef TCP_RFC7413 - if (tp->t_tfo_pending) { + if (IS_FASTOPEN(tp->t_flags) && tp->t_tfo_pending) { tcp_fastopen_decrement_counter(tp->t_tfo_pending); tp->t_tfo_pending = NULL; @@ -2470,7 +2478,6 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, * is retransmitted. */ if (!IS_FASTOPEN(tp->t_flags)) -#endif cc_conn_init(tp); tcp_timer_activate(tp, TT_KEEP, TP_KEEPIDLE(tp)); } @@ -3036,12 +3043,8 @@ dodata: /* XXX */ * case PRU_RCVD). If a FIN has already been received on this * connection then we just ignore the text. */ -#ifdef TCP_RFC7413 tfo_syn = ((tp->t_state == TCPS_SYN_RECEIVED) && IS_FASTOPEN(tp->t_flags)); -#else -#define tfo_syn (false) -#endif if ((tlen || (thflags & TH_FIN) || tfo_syn) && TCPS_HAVERCVDFIN(tp->t_state) == 0) { tcp_seq save_start = th->th_seq; @@ -3265,9 +3268,6 @@ drop: if (tp != NULL) INP_WUNLOCK(tp->t_inpcb); m_freem(m); -#ifndef TCP_RFC7413 -#undef tfo_syn -#endif } /* @@ -3421,21 +3421,21 @@ tcp_dooptions(struct tcpopt *to, u_char *cp, int cnt, int flags) to->to_sacks = cp + 2; TCPSTAT_INC(tcps_sack_rcv_blocks); break; -#ifdef TCP_RFC7413 case TCPOPT_FAST_OPEN: - if ((optlen != TCPOLEN_FAST_OPEN_EMPTY) && - (optlen < TCPOLEN_FAST_OPEN_MIN) && - (optlen > TCPOLEN_FAST_OPEN_MAX)) - continue; + /* + * Cookie length validation is performed by the + * server side cookie checking code or the client + * side cookie cache update code. + */ if (!(flags & TO_SYN)) continue; - if (!V_tcp_fastopen_enabled) + if (!V_tcp_fastopen_client_enable && + !V_tcp_fastopen_server_enable) continue; to->to_flags |= TOF_FASTOPEN; to->to_tfo_len = optlen - 2; to->to_tfo_cookie = to->to_tfo_len ? cp + 2 : NULL; break; -#endif default: continue; } diff --git a/freebsd/sys/netinet/tcp_log_buf.h b/freebsd/sys/netinet/tcp_log_buf.h new file mode 100644 index 00000000..58713fe5 --- /dev/null +++ b/freebsd/sys/netinet/tcp_log_buf.h @@ -0,0 +1,368 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2016-2018 + * Netflix Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __tcp_log_buf_h__ +#define __tcp_log_buf_h__ + +#define TCP_LOG_REASON_LEN 32 +#define TCP_LOG_BUF_VER (6) + +/* + * Because the (struct tcp_log_buffer) includes 8-byte uint64_t's, it requires + * 8-byte alignment to work properly on all platforms. Therefore, we will + * enforce 8-byte alignment for all the structures that may appear by + * themselves (instead of being embedded in another structure) in a data + * stream. + */ +#define ALIGN_TCP_LOG __aligned(8) + +/* Information about the socketbuffer state. */ +struct tcp_log_sockbuf +{ + uint32_t tls_sb_acc; /* available chars (sb->sb_acc) */ + uint32_t tls_sb_ccc; /* claimed chars (sb->sb_ccc) */ + uint32_t tls_sb_spare; /* spare */ +}; + +/* Optional, verbose information that may be appended to an event log. */ +struct tcp_log_verbose +{ +#define TCP_FUNC_LEN 32 + char tlv_snd_frm[TCP_FUNC_LEN]; /* tcp_output() caller */ + char tlv_trace_func[TCP_FUNC_LEN]; /* Function that + generated trace */ + uint32_t tlv_trace_line; /* Line number that generated trace */ + uint8_t _pad[4]; +} ALIGN_TCP_LOG; + +/* Internal RACK state variables. */ +struct tcp_log_rack +{ + uint32_t tlr_rack_rtt; /* rc_rack_rtt */ + uint8_t tlr_state; /* Internal RACK state */ + uint8_t _pad[3]; /* Padding */ +}; + +struct tcp_log_bbr { + uint64_t cur_del_rate; + uint64_t delRate; + uint64_t rttProp; + uint64_t bw_inuse; + uint32_t inflight; + uint32_t applimited; + uint32_t delivered; + uint32_t timeStamp; + uint32_t epoch; + uint32_t lt_epoch; + uint32_t pkts_out; + uint32_t flex1; + uint32_t flex2; + uint32_t flex3; + uint32_t flex4; + uint32_t flex5; + uint32_t flex6; + uint32_t lost; + uint16_t pacing_gain; + uint16_t cwnd_gain; + uint16_t flex7; + uint8_t bbr_state; + uint8_t bbr_substate; + uint8_t inpacer; + uint8_t ininput; + uint8_t use_lt_bw; + uint8_t flex8; + uint32_t pkt_epoch; +}; + +/* Per-stack stack-specific info. */ +union tcp_log_stackspecific +{ + struct tcp_log_rack u_rack; + struct tcp_log_bbr u_bbr; +}; + +struct tcp_log_buffer +{ + /* Event basics */ + struct timeval tlb_tv; /* Timestamp of trace */ + uint32_t tlb_ticks; /* Timestamp of trace */ + uint32_t tlb_sn; /* Serial number */ + uint8_t tlb_stackid; /* Stack ID */ + uint8_t tlb_eventid; /* Event ID */ + uint16_t tlb_eventflags; /* Flags for the record */ +#define TLB_FLAG_RXBUF 0x0001 /* Includes receive buffer info */ +#define TLB_FLAG_TXBUF 0x0002 /* Includes send buffer info */ +#define TLB_FLAG_HDR 0x0004 /* Includes a TCP header */ +#define TLB_FLAG_VERBOSE 0x0008 /* Includes function/line numbers */ +#define TLB_FLAG_STACKINFO 0x0010 /* Includes stack-specific info */ + int tlb_errno; /* Event error (if any) */ + + /* Internal session state */ + struct tcp_log_sockbuf tlb_rxbuf; /* Receive buffer */ + struct tcp_log_sockbuf tlb_txbuf; /* Send buffer */ + + int tlb_state; /* TCPCB t_state */ + uint32_t tlb_starttime; /* TCPCB t_starttime */ + uint32_t tlb_iss; /* TCPCB iss */ + uint32_t tlb_flags; /* TCPCB flags */ + uint32_t tlb_snd_una; /* TCPCB snd_una */ + uint32_t tlb_snd_max; /* TCPCB snd_max */ + uint32_t tlb_snd_cwnd; /* TCPCB snd_cwnd */ + uint32_t tlb_snd_nxt; /* TCPCB snd_nxt */ + uint32_t tlb_snd_recover;/* TCPCB snd_recover */ + uint32_t tlb_snd_wnd; /* TCPCB snd_wnd */ + uint32_t tlb_snd_ssthresh; /* TCPCB snd_ssthresh */ + uint32_t tlb_srtt; /* TCPCB t_srtt */ + uint32_t tlb_rttvar; /* TCPCB t_rttvar */ + uint32_t tlb_rcv_up; /* TCPCB rcv_up */ + uint32_t tlb_rcv_adv; /* TCPCB rcv_adv */ + uint32_t tlb_rcv_nxt; /* TCPCB rcv_nxt */ + tcp_seq tlb_sack_newdata; /* TCPCB sack_newdata */ + uint32_t tlb_rcv_wnd; /* TCPCB rcv_wnd */ + uint32_t tlb_dupacks; /* TCPCB t_dupacks */ + int tlb_segqlen; /* TCPCB segqlen */ + int tlb_snd_numholes; /* TCPCB snd_numholes */ + uint32_t tlb_flex1; /* Event specific information */ + uint32_t tlb_flex2; /* Event specific information */ + uint8_t tlb_snd_scale:4, /* TCPCB snd_scale */ + tlb_rcv_scale:4; /* TCPCB rcv_scale */ + uint8_t _pad[3]; /* Padding */ + + /* Per-stack info */ + union tcp_log_stackspecific tlb_stackinfo; +#define tlb_rack tlb_stackinfo.u_rack + + /* The packet */ + uint32_t tlb_len; /* The packet's data length */ + struct tcphdr tlb_th; /* The TCP header */ + uint8_t tlb_opts[TCP_MAXOLEN]; /* The TCP options */ + + /* Verbose information (optional) */ + struct tcp_log_verbose tlb_verbose[0]; +} ALIGN_TCP_LOG; + +enum tcp_log_events { + TCP_LOG_IN = 1, /* Incoming packet 1 */ + TCP_LOG_OUT, /* Transmit (without other event) 2 */ + TCP_LOG_RTO, /* Retransmit timeout 3 */ + TCP_LOG_TF_ACK, /* Transmit due to TF_ACK 4 */ + TCP_LOG_BAD_RETRAN, /* Detected bad retransmission 5 */ + TCP_LOG_PRR, /* Doing PRR 6 */ + TCP_LOG_REORDER,/* Detected reorder 7 */ + TCP_LOG_PACER, /* Pacer sending a packet 8 */ + BBR_LOG_BBRUPD, /* We updated BBR info 9 */ + BBR_LOG_BBRSND, /* We did a slot calculation and sending is done 10 */ + BBR_LOG_ACKCLEAR, /* A ack clears all outstanding 11 */ + BBR_LOG_INQUEUE, /* The tcb had a packet input to it 12 */ + BBR_LOG_TIMERSTAR, /* Start a timer 13 */ + BBR_LOG_TIMERCANC, /* Cancel a timer 14 */ + BBR_LOG_ENTREC, /* Entered recovery 15 */ + BBR_LOG_EXITREC, /* Exited recovery 16 */ + BBR_LOG_CWND, /* Cwnd change 17 */ + BBR_LOG_BWSAMP, /* LT B/W sample has been made 18 */ + BBR_LOG_MSGSIZE, /* We received a EMSGSIZE error 19 */ + BBR_LOG_BBRRTT, /* BBR RTT is updated 20 */ + BBR_LOG_JUSTRET, /* We just returned out of output 21 */ + BBR_LOG_STATE, /* A BBR state change occured 22 */ + BBR_LOG_PKT_EPOCH, /* A BBR packet epoch occured 23 */ + BBR_LOG_PERSIST, /* BBR changed to/from a persists 24 */ + TCP_LOG_FLOWEND, /* End of a flow 25 */ + BBR_LOG_RTO, /* BBR's timeout includes BBR info 26 */ + BBR_LOG_DOSEG_DONE, /* pacer do_segment completes 27 */ + BBR_LOG_EXIT_GAIN, /* pacer do_segment completes 28 */ + BBR_LOG_THRESH_CALC, /* Doing threshold calculation 29 */ + BBR_LOG_EXTRACWNDGAIN, /* Removed 30 */ + TCP_LOG_USERSEND, /* User level sends data 31 */ + UNUSED_32, /* Unused 32 */ + UNUSED_33, /* Unused 33 */ + BBR_LOG_TIME_EPOCH, /* A timed based Epoch occured 34 */ + BBR_LOG_TO_PROCESS, /* A to was processed 35 */ + BBR_LOG_BBRTSO, /* TSO update 36 */ + BBR_LOG_PACERDIAG, /* Pacer diag insert 37 */ + BBR_LOG_LOWGAIN, /* Low gain accounting 38 */ + BBR_LOG_PROGRESS, /* Progress timer event 39 */ + TCP_LOG_SOCKET_OPT, /* A socket option is set 40 */ + BBR_LOG_TIMERPREP, /* A BBR var to debug out TLP issues 41 */ + BBR_LOG_ENOBUF_JMP, /* We had a enobuf jump 42 */ + BBR_LOG_PACING_CALC, /* calc the pacing time 43 */ + BBR_LOG_RTT_SHRINKS, /* We had a log reduction of rttProp 44 */ + BBR_LOG_BW_RED_EV, /* B/W reduction events 45 */ + BBR_LOG_REDUCE, /* old bbr log reduce for 4.1 and earlier 46*/ + TCP_LOG_RTT, /* A rtt (in useconds) is being sampled and applied to the srtt algo 47 */ + BBR_LOG_SETTINGS_CHG, /* Settings changed for loss response 48 */ + TCP_LOG_END /* End (keep at end) 49 */ +}; + +enum tcp_log_states { + TCP_LOG_STATE_CLEAR = -1, /* Deactivate and clear tracing */ + TCP_LOG_STATE_OFF = 0, /* Pause */ + TCP_LOG_STATE_TAIL=1, /* Keep the trailing events */ + TCP_LOG_STATE_HEAD=2, /* Keep the leading events */ + TCP_LOG_STATE_HEAD_AUTO=3, /* Keep the leading events, and + automatically dump them to the + device */ + TCP_LOG_STATE_CONTINUAL=4, /* Continually dump the data when full */ + TCP_LOG_STATE_TAIL_AUTO=5, /* Keep the trailing events, and + automatically dump them when the + session ends */ +}; + +/* Use this if we don't know whether the operation succeeded. */ +#define ERRNO_UNK (-1) + +/* + * If the user included dev/tcp_log/tcp_log_dev.h, then include our private + * headers. Otherwise, there is no reason to pollute all the files with an + * additional include. + * + * This structure is aligned to an 8-byte boundary to match the alignment + * requirements of (struct tcp_log_buffer). + */ +#ifdef __tcp_log_dev_h__ +struct tcp_log_header { + struct tcp_log_common_header tlh_common; +#define tlh_version tlh_common.tlch_version +#define tlh_type tlh_common.tlch_type +#define tlh_length tlh_common.tlch_length + struct in_endpoints tlh_ie; + struct timeval tlh_offset; /* Uptime -> UTC offset */ + char tlh_id[TCP_LOG_ID_LEN]; + char tlh_reason[TCP_LOG_REASON_LEN]; + uint8_t tlh_af; + uint8_t _pad[7]; +} ALIGN_TCP_LOG; + +#ifdef _KERNEL +struct tcp_log_dev_log_queue { + struct tcp_log_dev_queue tldl_common; + char tldl_id[TCP_LOG_ID_LEN]; + char tldl_reason[TCP_LOG_REASON_LEN]; + struct in_endpoints tldl_ie; + struct tcp_log_stailq tldl_entries; + int tldl_count; + uint8_t tldl_af; +}; +#endif /* _KERNEL */ +#endif /* __tcp_log_dev_h__ */ + +#ifdef _KERNEL + +#define TCP_LOG_BUF_DEFAULT_SESSION_LIMIT 10000 +#define TCP_LOG_BUF_DEFAULT_GLOBAL_LIMIT 1000000 + +/* + * TCP_LOG_EVENT_VERBOSE: The same as TCP_LOG_EVENT, except it always + * tries to record verbose information. + */ +#define TCP_LOG_EVENT_VERBOSE(tp, th, rxbuf, txbuf, eventid, errornum, len, stackinfo, th_hostorder, tv) \ + do { \ + if (tp->t_logstate != TCP_LOG_STATE_OFF) \ + tcp_log_event_(tp, th, rxbuf, txbuf, eventid, \ + errornum, len, stackinfo, th_hostorder, \ + tp->t_output_caller, __func__, __LINE__, tv); \ + } while (0) + +/* + * TCP_LOG_EVENT: This is a macro so we can capture function/line + * information when needed. + * + * Prototype: + * TCP_LOG_EVENT(struct tcpcb *tp, struct tcphdr *th, struct sockbuf *rxbuf, + * struct sockbuf *txbuf, uint8_t eventid, int errornum, + * union tcp_log_stackspecific *stackinfo) + * + * tp is mandatory and must be write locked. + * th is optional; if present, it will appear in the record. + * rxbuf and txbuf are optional; if present, they will appear in the record. + * eventid is mandatory. + * errornum is mandatory (it indicates the success or failure of the + * operation associated with the event). + * len indicates the length of the packet. If no packet, use 0. + * stackinfo is optional; if present, it will appear in the record. + */ +#ifdef TCP_LOG_FORCEVERBOSE +#define TCP_LOG_EVENT TCP_LOG_EVENT_VERBOSE +#else +#define TCP_LOG_EVENT(tp, th, rxbuf, txbuf, eventid, errornum, len, stackinfo, th_hostorder) \ + do { \ + if (tcp_log_verbose) \ + TCP_LOG_EVENT_VERBOSE(tp, th, rxbuf, txbuf, \ + eventid, errornum, len, stackinfo, \ + th_hostorder, NULL); \ + else if (tp->t_logstate != TCP_LOG_STATE_OFF) \ + tcp_log_event_(tp, th, rxbuf, txbuf, eventid, \ + errornum, len, stackinfo, th_hostorder, \ + NULL, NULL, 0, NULL); \ + } while (0) +#endif /* TCP_LOG_FORCEVERBOSE */ +#define TCP_LOG_EVENTP(tp, th, rxbuf, txbuf, eventid, errornum, len, stackinfo, th_hostorder, tv) \ + do { \ + if (tp->t_logstate != TCP_LOG_STATE_OFF) \ + tcp_log_event_(tp, th, rxbuf, txbuf, eventid, \ + errornum, len, stackinfo, th_hostorder, \ + NULL, NULL, 0, tv); \ + } while (0) + + +#ifdef TCP_BLACKBOX +extern bool tcp_log_verbose; +void tcp_log_drain(struct tcpcb *tp); +int tcp_log_dump_tp_logbuf(struct tcpcb *tp, char *reason, int how, bool force); +void tcp_log_dump_tp_bucket_logbufs(struct tcpcb *tp, char *reason); +struct tcp_log_buffer *tcp_log_event_(struct tcpcb *tp, struct tcphdr *th, struct sockbuf *rxbuf, + struct sockbuf *txbuf, uint8_t eventid, int errornum, uint32_t len, + union tcp_log_stackspecific *stackinfo, int th_hostorder, + const char *output_caller, const char *func, int line, const struct timeval *tv); +size_t tcp_log_get_id(struct tcpcb *tp, char *buf); +u_int tcp_log_get_id_cnt(struct tcpcb *tp); +int tcp_log_getlogbuf(struct sockopt *sopt, struct tcpcb *tp); +void tcp_log_init(void); +int tcp_log_set_id(struct tcpcb *tp, char *id); +int tcp_log_state_change(struct tcpcb *tp, int state); +void tcp_log_tcpcbinit(struct tcpcb *tp); +void tcp_log_tcpcbfini(struct tcpcb *tp); +void tcp_log_flowend(struct tcpcb *tp); +#else /* !TCP_BLACKBOX */ +#define tcp_log_verbose (false) + +static inline struct tcp_log_buffer * +tcp_log_event_(struct tcpcb *tp, struct tcphdr *th, struct sockbuf *rxbuf, + struct sockbuf *txbuf, uint8_t eventid, int errornum, uint32_t len, + union tcp_log_stackspecific *stackinfo, int th_hostorder, + const char *output_caller, const char *func, int line, + const struct timeval *tv) +{ + + return (NULL); +} +#endif /* TCP_BLACKBOX */ + +#endif /* _KERNEL */ +#endif /* __tcp_log_buf_h__ */ diff --git a/freebsd/sys/netinet/tcp_lro.c b/freebsd/sys/netinet/tcp_lro.c index 50481b50..7242eb5c 100644 --- a/freebsd/sys/netinet/tcp_lro.c +++ b/freebsd/sys/netinet/tcp_lro.c @@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$"); #include <netinet/ip.h> #include <netinet/ip_var.h> #include <netinet/tcp.h> +#include <netinet/tcp_seq.h> #include <netinet/tcp_lro.h> #include <netinet/tcp_var.h> @@ -796,7 +797,9 @@ tcp_lro_rx2(struct lro_ctrl *lc, struct mbuf *m, uint32_t csum, int use_hash) /* Try to append the new segment. */ if (__predict_false(seq != le->next_seq || - (tcp_data_len == 0 && le->ack_seq == th->th_ack))) { + (tcp_data_len == 0 && + le->ack_seq == th->th_ack && + le->window == th->th_win))) { /* Out of order packet or duplicate ACK. */ tcp_lro_active_remove(le); tcp_lro_flush(lc, le); @@ -813,12 +816,20 @@ tcp_lro_rx2(struct lro_ctrl *lc, struct mbuf *m, uint32_t csum, int use_hash) le->tsval = tsval; le->tsecr = *(ts_ptr + 2); } - - le->next_seq += tcp_data_len; - le->ack_seq = th->th_ack; - le->window = th->th_win; - le->append_cnt++; - + if (tcp_data_len || SEQ_GT(ntohl(th->th_ack), ntohl(le->ack_seq))) { + le->next_seq += tcp_data_len; + le->ack_seq = th->th_ack; + le->window = th->th_win; + le->append_cnt++; + } else if (th->th_ack == le->ack_seq) { + le->window = WIN_MAX(le->window, th->th_win); + le->append_cnt++; + } else { + /* no data and old ack */ + le->append_cnt++; + m_freem(m); + return (0); + } #ifdef TCP_LRO_UPDATE_CSUM le->ulp_csum += tcp_lro_rx_csum_fixup(le, l3hdr, th, tcp_data_len, ~csum); diff --git a/freebsd/sys/netinet/tcp_output.c b/freebsd/sys/netinet/tcp_output.c index d0f08e3a..8762407f 100644 --- a/freebsd/sys/netinet/tcp_output.c +++ b/freebsd/sys/netinet/tcp_output.c @@ -73,17 +73,16 @@ __FBSDID("$FreeBSD$"); #include <netinet/ip6.h> #include <netinet6/ip6_var.h> #endif -#ifdef TCP_RFC7413 -#include <netinet/tcp_fastopen.h> -#endif #include <netinet/tcp.h> #define TCPOUTFLAGS #include <netinet/tcp_fsm.h> +#include <netinet/tcp_log_buf.h> #include <netinet/tcp_seq.h> #include <netinet/tcp_timer.h> #include <netinet/tcp_var.h> #include <netinet/tcpip.h> #include <netinet/cc/cc.h> +#include <netinet/tcp_fastopen.h> #ifdef TCPPCAP #include <netinet/tcp_pcap.h> #endif @@ -214,6 +213,8 @@ tcp_output(struct tcpcb *tp) struct sackhole *p; int tso, mtu; struct tcpopt to; + unsigned int wanted_cookie = 0; + unsigned int dont_sendalot = 0; #if 0 int maxburst = TCP_MAXBURST; #endif @@ -231,7 +232,6 @@ tcp_output(struct tcpcb *tp) return (tcp_offload_output(tp)); #endif -#ifdef TCP_RFC7413 /* * For TFO connections in SYN_RECEIVED, only allow the initial * SYN|ACK and those sent by the retransmit timer. @@ -239,9 +239,9 @@ tcp_output(struct tcpcb *tp) if (IS_FASTOPEN(tp->t_flags) && (tp->t_state == TCPS_SYN_RECEIVED) && SEQ_GT(tp->snd_max, tp->snd_una) && /* initial SYN|ACK sent */ - (tp->snd_nxt != tp->snd_una)) /* not a retransmit */ + (tp->snd_nxt != tp->snd_una)) /* not a retransmit */ return (0); -#endif + /* * Determine length of data that should be transmitted, * and flags that will be used. @@ -427,7 +427,6 @@ after_sack_rexmit: if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una)) { if (tp->t_state != TCPS_SYN_RECEIVED) flags &= ~TH_SYN; -#ifdef TCP_RFC7413 /* * When sending additional segments following a TFO SYN|ACK, * do not include the SYN bit. @@ -435,7 +434,6 @@ after_sack_rexmit: if (IS_FASTOPEN(tp->t_flags) && (tp->t_state == TCPS_SYN_RECEIVED)) flags &= ~TH_SYN; -#endif off--, len++; } @@ -449,17 +447,24 @@ after_sack_rexmit: flags &= ~TH_FIN; } -#ifdef TCP_RFC7413 /* - * When retransmitting SYN|ACK on a passively-created TFO socket, - * don't include data, as the presence of data may have caused the - * original SYN|ACK to have been dropped by a middlebox. + * On TFO sockets, ensure no data is sent in the following cases: + * + * - When retransmitting SYN|ACK on a passively-created socket + * + * - When retransmitting SYN on an actively created socket + * + * - When sending a zero-length cookie (cookie request) on an + * actively created socket + * + * - When the socket is in the CLOSED state (RST is being sent) */ if (IS_FASTOPEN(tp->t_flags) && - (((tp->t_state == TCPS_SYN_RECEIVED) && (tp->t_rxtshift > 0)) || + (((flags & TH_SYN) && (tp->t_rxtshift > 0)) || + ((tp->t_state == TCPS_SYN_SENT) && + (tp->t_tfo_client_cookie_len == 0)) || (flags & TH_RST))) len = 0; -#endif if (len <= 0) { /* * If FIN has been sent but not acked, @@ -543,7 +548,7 @@ after_sack_rexmit: if ((tp->t_flags & TF_TSO) && V_tcp_do_tso && len > tp->t_maxseg && ((tp->t_flags & TF_SIGNATURE) == 0) && tp->rcv_numsacks == 0 && sack_rxmit == 0 && - ipoptlen == 0) + ipoptlen == 0 && !(flags & TH_SYN)) tso = 1; if (sack_rxmit) { @@ -763,22 +768,39 @@ send: tp->snd_nxt = tp->iss; to.to_mss = tcp_mssopt(&tp->t_inpcb->inp_inc); to.to_flags |= TOF_MSS; -#ifdef TCP_RFC7413 + /* - * Only include the TFO option on the first - * transmission of the SYN|ACK on a - * passively-created TFO socket, as the presence of - * the TFO option may have caused the original - * SYN|ACK to have been dropped by a middlebox. + * On SYN or SYN|ACK transmits on TFO connections, + * only include the TFO option if it is not a + * retransmit, as the presence of the TFO option may + * have caused the original SYN or SYN|ACK to have + * been dropped by a middlebox. */ if (IS_FASTOPEN(tp->t_flags) && - (tp->t_state == TCPS_SYN_RECEIVED) && (tp->t_rxtshift == 0)) { - to.to_tfo_len = TCP_FASTOPEN_COOKIE_LEN; - to.to_tfo_cookie = (u_char *)&tp->t_tfo_cookie; - to.to_flags |= TOF_FASTOPEN; + if (tp->t_state == TCPS_SYN_RECEIVED) { + to.to_tfo_len = TCP_FASTOPEN_COOKIE_LEN; + to.to_tfo_cookie = + (u_int8_t *)&tp->t_tfo_cookie.server; + to.to_flags |= TOF_FASTOPEN; + wanted_cookie = 1; + } else if (tp->t_state == TCPS_SYN_SENT) { + to.to_tfo_len = + tp->t_tfo_client_cookie_len; + to.to_tfo_cookie = + tp->t_tfo_cookie.client; + to.to_flags |= TOF_FASTOPEN; + wanted_cookie = 1; + /* + * If we wind up having more data to + * send with the SYN than can fit in + * one segment, don't send any more + * until the SYN|ACK comes back from + * the other end. + */ + dont_sendalot = 1; + } } -#endif } /* Window scaling. */ if ((flags & TH_SYN) && (tp->t_flags & TF_REQ_SCALE)) { @@ -822,6 +844,13 @@ send: /* Processing the options. */ hdrlen += optlen = tcp_addoptions(&to, opt); + /* + * If we wanted a TFO option to be added, but it was unable + * to fit, ensure no data is sent. + */ + if (IS_FASTOPEN(tp->t_flags) && wanted_cookie && + !(to.to_flags & TOF_FASTOPEN)) + len = 0; } /* @@ -966,6 +995,8 @@ send: } else { len = tp->t_maxseg - optlen - ipoptlen; sendalot = 1; + if (dont_sendalot) + sendalot = 0; } } else tso = 0; @@ -1282,6 +1313,10 @@ send: } #endif + /* We're getting ready to send; log now. */ + TCP_LOG_EVENT(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_OUT, ERRNO_UNK, + len, NULL, false); + /* * Enable TSO and specify the size of the segments. * The TCP pseudo header checksum is always provided. @@ -1521,6 +1556,9 @@ timer: } if (error) { + /* Record the error. */ + TCP_LOG_EVENT(tp, NULL, &so->so_rcv, &so->so_snd, TCP_LOG_OUT, + error, 0, NULL, false); /* * We know that the packet was lost, so back out the @@ -1769,15 +1807,16 @@ tcp_addoptions(struct tcpopt *to, u_char *optp) TCPSTAT_INC(tcps_sack_send_blocks); break; } -#ifdef TCP_RFC7413 case TOF_FASTOPEN: { int total_len; /* XXX is there any point to aligning this option? */ total_len = TCPOLEN_FAST_OPEN_EMPTY + to->to_tfo_len; - if (TCP_MAXOLEN - optlen < total_len) + if (TCP_MAXOLEN - optlen < total_len) { + to->to_flags &= ~TOF_FASTOPEN; continue; + } *optp++ = TCPOPT_FAST_OPEN; *optp++ = total_len; if (to->to_tfo_len > 0) { @@ -1787,7 +1826,6 @@ tcp_addoptions(struct tcpopt *to, u_char *optp) optlen += total_len; break; } -#endif default: panic("%s: unknown TCP option type", __func__); break; diff --git a/freebsd/sys/netinet/tcp_seq.h b/freebsd/sys/netinet/tcp_seq.h index ee1e1bfd..b29ae2aa 100644 --- a/freebsd/sys/netinet/tcp_seq.h +++ b/freebsd/sys/netinet/tcp_seq.h @@ -47,6 +47,14 @@ #define SEQ_MIN(a, b) ((SEQ_LT(a, b)) ? (a) : (b)) #define SEQ_MAX(a, b) ((SEQ_GT(a, b)) ? (a) : (b)) +#define WIN_LT(a,b) ((short)(ntohs(a)-ntohs(b)) < 0) +#define WIN_LEQ(a,b) ((short)(ntohs(a)-ntohs(b)) <= 0) +#define WIN_GT(a,b) ((short)(ntohs(a)-ntohs(b)) > 0) +#define WIN_GEQ(a,b) ((short)(ntohs(a)-ntohs(b)) >= 0) + +#define WIN_MIN(a, b) ((WIN_LT(a, b)) ? (a) : (b)) +#define WIN_MAX(a, b) ((WIN_GT(a, b)) ? (a) : (b)) + /* for modulo comparisons of timestamps */ #define TSTMP_LT(a,b) ((int)((a)-(b)) < 0) #define TSTMP_GT(a,b) ((int)((a)-(b)) > 0) diff --git a/freebsd/sys/netinet/tcp_subr.c b/freebsd/sys/netinet/tcp_subr.c index d9c0b57e..1b19aecb 100644 --- a/freebsd/sys/netinet/tcp_subr.c +++ b/freebsd/sys/netinet/tcp_subr.c @@ -99,20 +99,19 @@ __FBSDID("$FreeBSD$"); #include <netinet6/nd6.h> #endif -#ifdef TCP_RFC7413 -#include <netinet/tcp_fastopen.h> -#endif #include <netinet/tcp.h> #include <netinet/tcp_fsm.h> #include <netinet/tcp_seq.h> #include <netinet/tcp_timer.h> #include <netinet/tcp_var.h> +#include <netinet/tcp_log_buf.h> #include <netinet/tcp_syncache.h> #include <netinet/cc/cc.h> #ifdef INET6 #include <netinet6/tcp6_var.h> #endif #include <netinet/tcpip.h> +#include <netinet/tcp_fastopen.h> #ifdef TCPPCAP #include <netinet/tcp_pcap.h> #endif @@ -434,6 +433,71 @@ SYSCTL_PROC(_net_inet_tcp, OID_AUTO, functions_available, "list available TCP Function sets"); /* + * Exports one (struct tcp_function_id) for each non-alias. + */ +static int +sysctl_net_inet_list_func_ids(SYSCTL_HANDLER_ARGS) +{ + int error, cnt; + struct tcp_function *f; + struct tcp_function_id tfi; + + /* + * We don't allow writes. + */ + if (req->newptr != NULL) + return (EINVAL); + + /* + * Wire the old buffer so we can directly copy the functions to + * user space without dropping the lock. + */ + if (req->oldptr != NULL) { + error = sysctl_wire_old_buffer(req, 0); + if (error) + return (error); + } + + /* + * Walk the list, comparing the name of the function entry and + * function block to determine which is an alias. + * If exporting the list, copy out matching entries. Otherwise, + * just record the total length. + */ + cnt = 0; + rw_rlock(&tcp_function_lock); + TAILQ_FOREACH(f, &t_functions, tf_next) { + if (strncmp(f->tf_name, f->tf_fb->tfb_tcp_block_name, + TCP_FUNCTION_NAME_LEN_MAX)) + continue; + if (req->oldptr != NULL) { + tfi.tfi_id = f->tf_fb->tfb_id; + (void)strncpy(tfi.tfi_name, f->tf_name, + TCP_FUNCTION_NAME_LEN_MAX); + tfi.tfi_name[TCP_FUNCTION_NAME_LEN_MAX - 1] = '\0'; + error = SYSCTL_OUT(req, &tfi, sizeof(tfi)); + /* + * Don't stop on error, as that is the + * mechanism we use to accumulate length + * information if the buffer was too short. + */ + } else + cnt++; + } + rw_runlock(&tcp_function_lock); + if (req->oldptr == NULL) + error = SYSCTL_OUT(req, NULL, + (cnt + 1) * sizeof(struct tcp_function_id)); + + return (error); +} + +SYSCTL_PROC(_net_inet_tcp, OID_AUTO, function_ids, + CTLTYPE_OPAQUE | CTLFLAG_SKIP | CTLFLAG_RD | CTLFLAG_MPSAFE, + NULL, 0, sysctl_net_inet_list_func_ids, "S,tcp_function_id", + "List TCP function block name-to-ID mappings"); + +/* * Target size of TCP PCB hash tables. Must be a power of two. * * Note that this can be overridden by the kernel environment @@ -512,6 +576,8 @@ maketcp_hashsize(int size) return (hashsize); } +static volatile int next_tcp_stack_id = 1; + /* * Register a TCP function block with the name provided in the names * array. (Note that this function does NOT automatically register @@ -571,6 +637,7 @@ register_tcp_functions_as_names(struct tcp_function_block *blk, int wait, refcount_init(&blk->tfb_refcnt, 0); blk->tfb_flags = 0; + blk->tfb_id = atomic_fetchadd_int(&next_tcp_stack_id, 1); for (i = 0; i < *num_names; i++) { n = malloc(sizeof(struct tcp_function), M_TCPFUNCTIONS, wait); if (n == NULL) { @@ -761,9 +828,7 @@ tcp_init(void) V_sack_hole_zone = uma_zcreate("sackhole", sizeof(struct sackhole), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); -#ifdef TCP_RFC7413 tcp_fastopen_init(); -#endif /* Skip initialization of globals for non-default instances. */ if (!IS_DEFAULT_VNET(curvnet)) @@ -789,6 +854,10 @@ tcp_init(void) /* Setup the tcp function block list */ init_tcp_functions(); register_tcp_functions(&tcp_def_funcblk, M_WAITOK); +#ifdef TCP_BLACKBOX + /* Initialize the TCP logging data. */ + tcp_log_init(); +#endif if (tcp_soreceive_stream) { #ifdef INET @@ -850,13 +919,11 @@ tcp_destroy(void *unused __unused) uma_zdestroy(V_sack_hole_zone); uma_zdestroy(V_tcpcb_zone); -#ifdef TCP_RFC7413 /* * Cannot free the zone until all tcpcbs are released as we attach * the allocations to them. */ tcp_fastopen_destroy(); -#endif #ifdef TCP_HHOOK error = hhook_head_deregister(V_tcp_hhh[HHOOK_TCP_EST_IN]); @@ -1372,6 +1439,10 @@ tcp_newtcpcb(struct inpcb *inp) */ tcp_pcap_tcpcb_init(tp); #endif +#ifdef TCP_BLACKBOX + /* Initialize the per-TCPCB log data. */ + tcp_log_tcpcbinit(tp); +#endif if (tp->t_fb->tfb_tcp_fb_init) { (*tp->t_fb->tfb_tcp_fb_init)(tp); } @@ -1589,6 +1660,9 @@ tcp_discardcb(struct tcpcb *tp) inp->inp_ppcb = NULL; if (tp->t_timers->tt_draincnt == 0) { /* We own the last reference on tcpcb, let's free it. */ +#ifdef TCP_BLACKBOX + tcp_log_tcpcbfini(tp); +#endif TCPSTATES_DEC(tp->t_state); if (tp->t_fb->tfb_tcp_fb_fini) (*tp->t_fb->tfb_tcp_fb_fini)(tp, 1); @@ -1619,6 +1693,9 @@ tcp_timer_discard(void *ptp) tp->t_timers->tt_draincnt--; if (tp->t_timers->tt_draincnt == 0) { /* We own the last reference on this tcpcb, let's free it. */ +#ifdef TCP_BLACKBOX + tcp_log_tcpcbfini(tp); +#endif TCPSTATES_DEC(tp->t_state); if (tp->t_fb->tfb_tcp_fb_fini) (*tp->t_fb->tfb_tcp_fb_fini)(tp, 1); @@ -1653,7 +1730,6 @@ tcp_close(struct tcpcb *tp) if (tp->t_state == TCPS_LISTEN) tcp_offload_listen_stop(tp); #endif -#ifdef TCP_RFC7413 /* * This releases the TFO pending counter resource for TFO listen * sockets as well as passively-created TFO sockets that transition @@ -1663,7 +1739,6 @@ tcp_close(struct tcpcb *tp) tcp_fastopen_decrement_counter(tp->t_tfo_pending); tp->t_tfo_pending = NULL; } -#endif in_pcbdrop(inp); TCPSTAT_INC(tcps_closed); if (tp->t_state != TCPS_CLOSED) @@ -1714,6 +1789,9 @@ tcp_drain(void) if ((tcpb = intotcpcb(inpb)) != NULL) { tcp_reass_flush(tcpb); tcp_clean_sackreport(tcpb); +#ifdef TCP_BLACKBOX + tcp_log_drain(tcpb); +#endif #ifdef TCPPCAP if (tcp_pcap_aggressive_free) { /* Free the TCP PCAP queues. */ @@ -2413,6 +2491,9 @@ tcp_drop_syn_sent(struct inpcb *inp, int errno) if (tp->t_state != TCPS_SYN_SENT) return (inp); + if (IS_FASTOPEN(tp->t_flags)) + tcp_fastopen_disable_path(tp); + tp = tcp_drop(tp, errno); if (tp != NULL) return (inp); @@ -2867,6 +2948,7 @@ tcp_inptoxtp(const struct inpcb *inp, struct xtcpcb *xt) xt->t_state = TCPS_TIME_WAIT; } else { xt->t_state = tp->t_state; + xt->t_logstate = tp->t_logstate; xt->t_flags = tp->t_flags; xt->t_sndzerowin = tp->t_sndzerowin; xt->t_sndrexmitpack = tp->t_sndrexmitpack; @@ -2890,6 +2972,10 @@ tcp_inptoxtp(const struct inpcb *inp, struct xtcpcb *xt) bcopy(tp->t_fb->tfb_tcp_block_name, xt->xt_stack, TCP_FUNCTION_NAME_LEN_MAX); + bzero(xt->xt_logid, TCP_LOG_ID_LEN); +#ifdef TCP_BLACKBOX + (void)tcp_log_get_id(tp, xt->xt_logid); +#endif } xt->xt_len = sizeof(struct xtcpcb); diff --git a/freebsd/sys/netinet/tcp_syncache.c b/freebsd/sys/netinet/tcp_syncache.c index 0dc4a0a5..27e0e25f 100644 --- a/freebsd/sys/netinet/tcp_syncache.c +++ b/freebsd/sys/netinet/tcp_syncache.c @@ -85,9 +85,7 @@ __FBSDID("$FreeBSD$"); #include <netinet6/in6_pcb.h> #endif #include <netinet/tcp.h> -#ifdef TCP_RFC7413 #include <netinet/tcp_fastopen.h> -#endif #include <netinet/tcp_fsm.h> #include <netinet/tcp_seq.h> #include <netinet/tcp_timer.h> @@ -699,6 +697,8 @@ syncache_socket(struct syncache *sc, struct socket *lso, struct mbuf *m) inp->inp_inc.inc_flags = sc->sc_inc.inc_flags; #ifdef INET6 if (sc->sc_inc.inc_flags & INC_ISIPV6) { + inp->inp_vflag &= ~INP_IPV4; + inp->inp_vflag |= INP_IPV6; inp->in6p_laddr = sc->sc_inc.inc6_laddr; } else { inp->inp_vflag &= ~INP_IPV6; @@ -1186,7 +1186,6 @@ failed: return (0); } -#ifdef TCP_RFC7413 static void syncache_tfo_expand(struct syncache *sc, struct socket **lsop, struct mbuf *m, uint64_t response_cookie) @@ -1211,14 +1210,13 @@ syncache_tfo_expand(struct syncache *sc, struct socket **lsop, struct mbuf *m, inp = sotoinpcb(*lsop); tp = intotcpcb(inp); tp->t_flags |= TF_FASTOPEN; - tp->t_tfo_cookie = response_cookie; + tp->t_tfo_cookie.server = response_cookie; tp->snd_max = tp->iss; tp->snd_nxt = tp->iss; tp->t_tfo_pending = pending_counter; TCPSTAT_INC(tcps_sc_completed); } } -#endif /* TCP_RFC7413 */ /* * Given a LISTEN socket and an inbound SYN request, add @@ -1261,12 +1259,10 @@ syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, #endif struct syncache scs; struct ucred *cred; -#ifdef TCP_RFC7413 uint64_t tfo_response_cookie; unsigned int *tfo_pending = NULL; int tfo_cookie_valid = 0; int tfo_response_cookie_valid = 0; -#endif INP_WLOCK_ASSERT(inp); /* listen socket */ KASSERT((th->th_flags & (TH_RST|TH_ACK|TH_SYN)) == TH_SYN, @@ -1291,9 +1287,9 @@ syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, win = so->sol_sbrcv_hiwat; ltflags = (tp->t_flags & (TF_NOOPT | TF_SIGNATURE)); -#ifdef TCP_RFC7413 - if (V_tcp_fastopen_enabled && IS_FASTOPEN(tp->t_flags) && - (tp->t_tfo_pending != NULL) && (to->to_flags & TOF_FASTOPEN)) { + if (V_tcp_fastopen_server_enable && IS_FASTOPEN(tp->t_flags) && + (tp->t_tfo_pending != NULL) && + (to->to_flags & TOF_FASTOPEN)) { /* * Limit the number of pending TFO connections to * approximately half of the queue limit. This prevents TFO @@ -1317,7 +1313,6 @@ syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, */ tfo_pending = tp->t_tfo_pending; } -#endif /* By the time we drop the lock these should no longer be used. */ so = NULL; @@ -1330,9 +1325,7 @@ syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, } else mac_syncache_create(maclabel, inp); #endif -#ifdef TCP_RFC7413 if (!tfo_cookie_valid) -#endif INP_WUNLOCK(inp); /* @@ -1378,10 +1371,8 @@ syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, sc = syncache_lookup(inc, &sch); /* returns locked entry */ SCH_LOCK_ASSERT(sch); if (sc != NULL) { -#ifdef TCP_RFC7413 if (tfo_cookie_valid) INP_WUNLOCK(inp); -#endif TCPSTAT_INC(tcps_sc_dupsyn); if (ipopts) { /* @@ -1424,13 +1415,11 @@ syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, goto done; } -#ifdef TCP_RFC7413 if (tfo_cookie_valid) { bzero(&scs, sizeof(scs)); sc = &scs; goto skip_alloc; } -#endif sc = uma_zalloc(V_tcp_syncache.zone, M_NOWAIT | M_ZERO); if (sc == NULL) { @@ -1458,11 +1447,9 @@ syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, } } -#ifdef TCP_RFC7413 skip_alloc: if (!tfo_cookie_valid && tfo_response_cookie_valid) sc->sc_tfo_cookie = &tfo_response_cookie; -#endif /* * Fill in the syncache values. @@ -1571,14 +1558,12 @@ skip_alloc: #endif SCH_UNLOCK(sch); -#ifdef TCP_RFC7413 if (tfo_cookie_valid) { syncache_tfo_expand(sc, lsop, m, tfo_response_cookie); /* INP_WUNLOCK(inp) will be performed by the caller */ rv = 1; goto tfo_expanded; } -#endif /* * Do a standard 3-way handshake. @@ -1601,7 +1586,6 @@ done: *lsop = NULL; m_freem(m); } -#ifdef TCP_RFC7413 /* * If tfo_pending is not NULL here, then a TFO SYN that did not * result in a new socket was processed and the associated pending @@ -1612,7 +1596,6 @@ done: tcp_fastopen_decrement_counter(tfo_pending); tfo_expanded: -#endif if (cred != NULL) crfree(cred); #ifdef MAC @@ -1749,7 +1732,6 @@ syncache_respond(struct syncache *sc, struct syncache_head *sch, int locked, if (sc->sc_flags & SCF_SIGNATURE) to.to_flags |= TOF_SIGNATURE; #endif -#ifdef TCP_RFC7413 if (sc->sc_tfo_cookie) { to.to_flags |= TOF_FASTOPEN; to.to_tfo_len = TCP_FASTOPEN_COOKIE_LEN; @@ -1757,7 +1739,6 @@ syncache_respond(struct syncache *sc, struct syncache_head *sch, int locked, /* don't send cookie again when retransmitting response */ sc->sc_tfo_cookie = NULL; } -#endif optlen = tcp_addoptions(&to, (u_char *)(th + 1)); /* Adjust headers by option size. */ diff --git a/freebsd/sys/netinet/tcp_syncache.h b/freebsd/sys/netinet/tcp_syncache.h index d9ebb65c..92a7c7c9 100644 --- a/freebsd/sys/netinet/tcp_syncache.h +++ b/freebsd/sys/netinet/tcp_syncache.h @@ -75,9 +75,7 @@ struct syncache { #endif struct label *sc_label; /* MAC label reference */ struct ucred *sc_cred; /* cred cache for jail checks */ -#ifdef TCP_RFC7413 void *sc_tfo_cookie; /* for TCP Fast Open response */ -#endif void *sc_pspare; /* TCP_SIGNATURE */ u_int32_t sc_spare[2]; /* UTO */ }; diff --git a/freebsd/sys/netinet/tcp_timer.c b/freebsd/sys/netinet/tcp_timer.c index 539913f4..69bd052b 100644 --- a/freebsd/sys/netinet/tcp_timer.c +++ b/freebsd/sys/netinet/tcp_timer.c @@ -70,6 +70,7 @@ __FBSDID("$FreeBSD$"); #include <netinet/ip_var.h> #include <netinet/tcp.h> #include <netinet/tcp_fsm.h> +#include <netinet/tcp_log_buf.h> #include <netinet/tcp_timer.h> #include <netinet/tcp_var.h> #include <netinet/cc/cc.h> @@ -646,6 +647,7 @@ tcp_timer_rexmt(void * xtp) KASSERT((tp->t_timers->tt_flags & TT_STOPPED) == 0, ("%s: tp %p tcpcb can't be stopped here", __func__, tp)); tcp_free_sackholes(tp); + TCP_LOG_EVENT(tp, NULL, NULL, NULL, TCP_LOG_RTO, 0, 0, NULL, false); if (tp->t_fb->tfb_tcp_rexmit_tmr) { /* The stack has a timer action too. */ (*tp->t_fb->tfb_tcp_rexmit_tmr)(tp); diff --git a/freebsd/sys/netinet/tcp_timewait.c b/freebsd/sys/netinet/tcp_timewait.c index 61263dfe..aa26cb37 100644 --- a/freebsd/sys/netinet/tcp_timewait.c +++ b/freebsd/sys/netinet/tcp_timewait.c @@ -174,7 +174,7 @@ SYSCTL_PROC(_net_inet_tcp, OID_AUTO, maxtcptw, CTLTYPE_INT|CTLFLAG_RW, &maxtcptw, 0, sysctl_maxtcptw, "IU", "Maximum number of compressed TCP TIME_WAIT entries"); -VNET_DEFINE(int, nolocaltimewait) = 0; +static VNET_DEFINE(int, nolocaltimewait) = 0; #define V_nolocaltimewait VNET(nolocaltimewait) SYSCTL_INT(_net_inet_tcp, OID_AUTO, nolocaltimewait, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(nolocaltimewait), 0, @@ -227,12 +227,12 @@ tcp_tw_destroy(void) void tcp_twstart(struct tcpcb *tp) { - struct tcptw *tw; + struct tcptw twlocal, *tw; struct inpcb *inp = tp->t_inpcb; - int acknow; struct socket *so; + bool acknow, local; #ifdef INET6 - int isipv6 = inp->inp_inc.inc_flags & INC_ISIPV6; + bool isipv6 = inp->inp_inc.inc_flags & INC_ISIPV6; #endif INP_INFO_RLOCK_ASSERT(&V_tcbinfo); @@ -243,25 +243,18 @@ tcp_twstart(struct tcpcb *tp) "(inp->inp_flags & INP_DROPPED) != 0")); if (V_nolocaltimewait) { - int error = 0; #ifdef INET6 if (isipv6) - error = in6_localaddr(&inp->in6p_faddr); -#endif -#if defined(INET6) && defined(INET) + local = in6_localaddr(&inp->in6p_faddr); else #endif #ifdef INET - error = in_localip(inp->inp_faddr); + local = in_localip(inp->inp_faddr); +#else + local = false; #endif - if (error) { - tp = tcp_close(tp); - if (tp != NULL) - INP_WUNLOCK(inp); - return; - } - } - + } else + local = false; /* * For use only by DTrace. We do not reference the state @@ -269,7 +262,10 @@ tcp_twstart(struct tcpcb *tp) */ tcp_state_change(tp, TCPS_TIME_WAIT); - tw = uma_zalloc(V_tcptw_zone, M_NOWAIT); + if (local) + tw = &twlocal; + else + tw = uma_zalloc(V_tcptw_zone, M_NOWAIT); if (tw == NULL) { /* * Reached limit on total number of TIMEWAIT connections @@ -288,11 +284,10 @@ tcp_twstart(struct tcpcb *tp) } } /* - * The tcptw will hold a reference on its inpcb until tcp_twclose - * is called + * For !local case the tcptw will hold a reference on its inpcb + * until tcp_twclose is called. */ tw->tw_inpcb = inp; - in_pcbref(inp); /* Reference from tw */ /* * Recover last window size sent. @@ -339,16 +334,19 @@ tcp_twstart(struct tcpcb *tp) tcp_discardcb(tp); so = inp->inp_socket; soisdisconnected(so); - tw->tw_cred = crhold(so->so_cred); - SOCK_LOCK(so); tw->tw_so_options = so->so_options; - SOCK_UNLOCK(so); + inp->inp_flags |= INP_TIMEWAIT; if (acknow) tcp_twrespond(tw, TH_ACK); - inp->inp_ppcb = tw; - inp->inp_flags |= INP_TIMEWAIT; - TCPSTATES_INC(TCPS_TIME_WAIT); - tcp_tw_2msl_reset(tw, 0); + if (local) + in_pcbdrop(inp); + else { + in_pcbref(inp); /* Reference from tw */ + tw->tw_cred = crhold(so->so_cred); + inp->inp_ppcb = tw; + TCPSTATES_INC(TCPS_TIME_WAIT); + tcp_tw_2msl_reset(tw, 0); + } /* * If the inpcb owns the sole reference to the socket, then we can diff --git a/freebsd/sys/netinet/tcp_usrreq.c b/freebsd/sys/netinet/tcp_usrreq.c index 76c6d952..c93b2d4a 100644 --- a/freebsd/sys/netinet/tcp_usrreq.c +++ b/freebsd/sys/netinet/tcp_usrreq.c @@ -87,16 +87,15 @@ __FBSDID("$FreeBSD$"); #include <netinet6/ip6_var.h> #include <netinet6/scope6_var.h> #endif -#ifdef TCP_RFC7413 -#include <netinet/tcp_fastopen.h> -#endif #include <netinet/tcp.h> #include <netinet/tcp_fsm.h> #include <netinet/tcp_seq.h> #include <netinet/tcp_timer.h> #include <netinet/tcp_var.h> +#include <netinet/tcp_log_buf.h> #include <netinet/tcpip.h> #include <netinet/cc/cc.h> +#include <netinet/tcp_fastopen.h> #ifdef TCPPCAP #include <netinet/tcp_pcap.h> #endif @@ -202,15 +201,18 @@ tcp_detach(struct socket *so, struct inpcb *inp) * XXXRW: Would it be cleaner to free the tcptw here? * * Astute question indeed, from twtcp perspective there are - * three cases to consider: + * four cases to consider: * * #1 tcp_detach is called at tcptw creation time by * tcp_twstart, then do not discard the newly created tcptw * and leave inpcb present until timewait ends - * #2 tcp_detach is called at timewait end (or reuse) by + * #2 tcp_detach is called at tcptw creation time by + * tcp_twstart, but connection is local and tw will be + * discarded immediately + * #3 tcp_detach is called at timewait end (or reuse) by * tcp_twclose, then the tcptw has already been discarded * (or reused) and inpcb is freed here - * #3 tcp_detach is called() after timewait ends (or reuse) + * #4 tcp_detach is called() after timewait ends (or reuse) * (e.g. by soclose), then tcptw has already been discarded * (or reused) and inpcb is freed here * @@ -432,10 +434,9 @@ tcp_usr_listen(struct socket *so, int backlog, struct thread *td) } SOCK_UNLOCK(so); -#ifdef TCP_RFC7413 if (IS_FASTOPEN(tp->t_flags)) tp->t_tfo_pending = tcp_fastopen_alloc_counter(); -#endif + out: TCPDEBUG2(PRU_LISTEN); TCP_PROBE2(debug__user, tp, PRU_LISTEN); @@ -482,10 +483,9 @@ tcp6_usr_listen(struct socket *so, int backlog, struct thread *td) } SOCK_UNLOCK(so); -#ifdef TCP_RFC7413 if (IS_FASTOPEN(tp->t_flags)) tp->t_tfo_pending = tcp_fastopen_alloc_counter(); -#endif + out: TCPDEBUG2(PRU_LISTEN); TCP_PROBE2(debug__user, tp, PRU_LISTEN); @@ -850,7 +850,6 @@ tcp_usr_rcvd(struct socket *so, int flags) } tp = intotcpcb(inp); TCPDEBUG1(); -#ifdef TCP_RFC7413 /* * For passively-created TFO connections, don't attempt a window * update while still in SYN_RECEIVED as this may trigger an early @@ -861,7 +860,6 @@ tcp_usr_rcvd(struct socket *so, int flags) if (IS_FASTOPEN(tp->t_flags) && (tp->t_state == TCPS_SYN_RECEIVED)) goto out; -#endif #ifdef TCP_OFFLOAD if (tp->t_flags & TF_TOE) tcp_offload_rcvd(tp); @@ -952,8 +950,12 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, #endif if (error) goto out; - tp->snd_wnd = TTCP_CLIENT_SND_WND; - tcp_mss(tp, -1); + if (IS_FASTOPEN(tp->t_flags)) + tcp_fastopen_connect(tp); + else { + tp->snd_wnd = TTCP_CLIENT_SND_WND; + tcp_mss(tp, -1); + } } if (flags & PRUS_EOF) { /* @@ -999,6 +1001,12 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, * initialize window to default value, and * initialize maxseg using peer's cached MSS. */ + + /* + * Not going to contemplate SYN|URG + */ + if (IS_FASTOPEN(tp->t_flags)) + tp->t_flags &= ~TF_FASTOPEN; #ifdef INET6 if (isipv6) error = tcp6_connect(tp, nam, td); @@ -1021,6 +1029,11 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, tp->t_flags &= ~TF_FORCEDATA; } } + TCP_LOG_EVENT(tp, NULL, + &inp->inp_socket->so_rcv, + &inp->inp_socket->so_snd, + TCP_LOG_USERSEND, error, + 0, NULL, false); out: TCPDEBUG2((flags & PRUS_OOB) ? PRU_SENDOOB : ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND)); @@ -1528,6 +1541,15 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) return (tp->t_fb->tfb_tcp_ctloutput(so, sopt, inp, tp)); } +/* + * If this assert becomes untrue, we need to change the size of the buf + * variable in tcp_default_ctloutput(). + */ +#ifdef CTASSERT +CTASSERT(TCP_CA_NAME_MAX <= TCP_LOG_ID_LEN); +CTASSERT(TCP_LOG_REASON_LEN <= TCP_LOG_ID_LEN); +#endif + int tcp_default_ctloutput(struct socket *so, struct sockopt *sopt, struct inpcb *inp, struct tcpcb *tp) { @@ -1535,7 +1557,7 @@ tcp_default_ctloutput(struct socket *so, struct sockopt *sopt, struct inpcb *inp u_int ui; struct tcp_info ti; struct cc_algo *algo; - char *pbuf, buf[TCP_CA_NAME_MAX]; + char *pbuf, buf[TCP_LOG_ID_LEN]; size_t len; /* @@ -1770,27 +1792,102 @@ unlock_and_done: goto unlock_and_done; #endif -#ifdef TCP_RFC7413 - case TCP_FASTOPEN: + case TCP_FASTOPEN: { + struct tcp_fastopen tfo_optval; + INP_WUNLOCK(inp); - if (!V_tcp_fastopen_enabled) + if (!V_tcp_fastopen_client_enable && + !V_tcp_fastopen_server_enable) return (EPERM); - error = sooptcopyin(sopt, &optval, sizeof optval, - sizeof optval); + error = sooptcopyin(sopt, &tfo_optval, + sizeof(tfo_optval), sizeof(int)); if (error) return (error); INP_WLOCK_RECHECK(inp); - if (optval) { - tp->t_flags |= TF_FASTOPEN; - if ((tp->t_state == TCPS_LISTEN) && - (tp->t_tfo_pending == NULL)) - tp->t_tfo_pending = - tcp_fastopen_alloc_counter(); + if (tfo_optval.enable) { + if (tp->t_state == TCPS_LISTEN) { + if (!V_tcp_fastopen_server_enable) { + error = EPERM; + goto unlock_and_done; + } + + tp->t_flags |= TF_FASTOPEN; + if (tp->t_tfo_pending == NULL) + tp->t_tfo_pending = + tcp_fastopen_alloc_counter(); + } else { + /* + * If a pre-shared key was provided, + * stash it in the client cookie + * field of the tcpcb for use during + * connect. + */ + if (sopt->sopt_valsize == + sizeof(tfo_optval)) { + memcpy(tp->t_tfo_cookie.client, + tfo_optval.psk, + TCP_FASTOPEN_PSK_LEN); + tp->t_tfo_client_cookie_len = + TCP_FASTOPEN_PSK_LEN; + } + tp->t_flags |= TF_FASTOPEN; + } } else tp->t_flags &= ~TF_FASTOPEN; goto unlock_and_done; + } + +#ifdef TCP_BLACKBOX + case TCP_LOG: + INP_WUNLOCK(inp); + error = sooptcopyin(sopt, &optval, sizeof optval, + sizeof optval); + if (error) + return (error); + + INP_WLOCK_RECHECK(inp); + error = tcp_log_state_change(tp, optval); + goto unlock_and_done; + + case TCP_LOGBUF: + INP_WUNLOCK(inp); + error = EINVAL; + break; + + case TCP_LOGID: + INP_WUNLOCK(inp); + error = sooptcopyin(sopt, buf, TCP_LOG_ID_LEN - 1, 0); + if (error) + break; + buf[sopt->sopt_valsize] = '\0'; + INP_WLOCK_RECHECK(inp); + error = tcp_log_set_id(tp, buf); + /* tcp_log_set_id() unlocks the INP. */ + break; + + case TCP_LOGDUMP: + case TCP_LOGDUMPID: + INP_WUNLOCK(inp); + error = + sooptcopyin(sopt, buf, TCP_LOG_REASON_LEN - 1, 0); + if (error) + break; + buf[sopt->sopt_valsize] = '\0'; + INP_WLOCK_RECHECK(inp); + if (sopt->sopt_name == TCP_LOGDUMP) { + error = tcp_log_dump_tp_logbuf(tp, buf, + M_WAITOK, true); + INP_WUNLOCK(inp); + } else { + tcp_log_dump_tp_bucket_logbufs(tp, buf); + /* + * tcp_log_dump_tp_bucket_logbufs() drops the + * INP lock. + */ + } + break; #endif default: @@ -1873,13 +1970,31 @@ unlock_and_done: error = sooptcopyout(sopt, &optval, sizeof optval); break; #endif - -#ifdef TCP_RFC7413 case TCP_FASTOPEN: optval = tp->t_flags & TF_FASTOPEN; INP_WUNLOCK(inp); error = sooptcopyout(sopt, &optval, sizeof optval); break; +#ifdef TCP_BLACKBOX + case TCP_LOG: + optval = tp->t_logstate; + INP_WUNLOCK(inp); + error = sooptcopyout(sopt, &optval, sizeof(optval)); + break; + case TCP_LOGBUF: + /* tcp_log_getlogbuf() does INP_WUNLOCK(inp) */ + error = tcp_log_getlogbuf(sopt, tp); + break; + case TCP_LOGID: + len = tcp_log_get_id(tp, buf); + INP_WUNLOCK(inp); + error = sooptcopyout(sopt, buf, len + 1); + break; + case TCP_LOGDUMP: + case TCP_LOGDUMPID: + INP_WUNLOCK(inp); + error = EINVAL; + break; #endif default: INP_WUNLOCK(inp); diff --git a/freebsd/sys/netinet/tcp_var.h b/freebsd/sys/netinet/tcp_var.h index 35b44410..f09bd19c 100644 --- a/freebsd/sys/netinet/tcp_var.h +++ b/freebsd/sys/netinet/tcp_var.h @@ -79,6 +79,8 @@ struct sackhint { uint64_t _pad[1]; /* TBD */ }; +STAILQ_HEAD(tcp_log_stailq, tcp_log_mem); + /* * Tcp control block, one per tcp; fields: * Organized for 16 byte cacheline efficiency. @@ -189,12 +191,21 @@ struct tcpcb { u_int t_tsomaxsegcount; /* TSO maximum segment count */ u_int t_tsomaxsegsize; /* TSO maximum segment size in bytes */ u_int t_flags2; /* More tcpcb flags storage */ + int t_logstate; /* State of "black box" logging */ + struct tcp_log_stailq t_logs; /* Log buffer */ + int t_lognum; /* Number of log entries */ + uint32_t t_logsn; /* Log "serial number" */ + struct tcp_log_id_node *t_lin; + struct tcp_log_id_bucket *t_lib; + const char *t_output_caller; /* Function that called tcp_output */ struct tcp_function_block *t_fb;/* TCP function call block */ void *t_fb_ptr; /* Pointer to t_fb specific data */ -#ifdef TCP_RFC7413 - uint64_t t_tfo_cookie; /* TCP Fast Open cookie */ - unsigned int *t_tfo_pending; /* TCP Fast Open pending counter */ -#endif + uint8_t t_tfo_client_cookie_len; /* TCP Fast Open client cookie length */ + unsigned int *t_tfo_pending; /* TCP Fast Open server pending counter */ + union { + uint8_t client[TCP_FASTOPEN_MAX_COOKIE_LEN]; + uint64_t server; + } t_tfo_cookie; /* TCP Fast Open cookie to send */ #ifdef TCPPCAP struct mbufq t_inpkts; /* List of saved input packets. */ struct mbufq t_outpkts; /* List of saved output packets. */ @@ -265,6 +276,7 @@ struct tcp_function_block { int (*tfb_tcp_handoff_ok)(struct tcpcb *); volatile uint32_t tfb_refcnt; uint32_t tfb_flags; + uint8_t tfb_id; }; struct tcp_function { @@ -337,11 +349,12 @@ TAILQ_HEAD(tcp_funchead, tcp_function); #define TCPOOB_HADDATA 0x02 /* - * Flags for PLPMTU handling, t_flags2 + * Flags for the extended TCP flags field, t_flags2 */ #define TF2_PLPMTU_BLACKHOLE 0x00000001 /* Possible PLPMTUD Black Hole. */ #define TF2_PLPMTU_PMTUD 0x00000002 /* Allowed to attempt PLPMTUD. */ #define TF2_PLPMTU_MAXSEGSNT 0x00000004 /* Last seg sent was full seg. */ +#define TF2_LOG_AUTO 0x00000008 /* Session is auto-logging. */ /* * Structure to hold TCP options that are only used during segment @@ -365,7 +378,7 @@ struct tcpopt { u_int32_t to_tsecr; /* reflected timestamp */ u_char *to_sacks; /* pointer to the first SACK blocks */ u_char *to_signature; /* pointer to the TCP-MD5 signature */ - u_char *to_tfo_cookie; /* pointer to the TFO cookie */ + u_int8_t *to_tfo_cookie; /* pointer to the TFO cookie */ u_int16_t to_mss; /* maximum segment size */ u_int8_t to_wscale; /* window scaling */ u_int8_t to_nsacks; /* number of SACK blocks */ @@ -652,6 +665,7 @@ struct xtcpcb { size_t xt_len; /* length of this structure */ struct xinpcb xt_inp; char xt_stack[TCP_FUNCTION_NAME_LEN_MAX]; /* (s) */ + char xt_logid[TCP_LOG_ID_LEN]; /* (s) */ int64_t spare64[8]; int32_t t_state; /* (s,p) */ uint32_t t_flags; /* (s,p) */ @@ -664,14 +678,24 @@ struct xtcpcb { int32_t tt_keep; /* (s) */ int32_t tt_2msl; /* (s) */ int32_t tt_delack; /* (s) */ + int32_t t_logstate; /* (3) */ int32_t spare32[32]; } __aligned(8); + #ifdef _KERNEL void tcp_inptoxtp(const struct inpcb *, struct xtcpcb *); #endif #endif /* + * TCP function name-to-id mapping exported to user-land via sysctl(3). + */ +struct tcp_function_id { + uint8_t tfi_id; + char tfi_name[TCP_FUNCTION_NAME_LEN_MAX]; +}; + +/* * Identifiers for TCP sysctl nodes */ #define TCPCTL_DO_RFC1323 1 /* use RFC-1323 extensions */ diff --git a/freebsd/sys/netinet6/dest6.c b/freebsd/sys/netinet6/dest6.c index 1c9efc25..50a836ba 100644 --- a/freebsd/sys/netinet6/dest6.c +++ b/freebsd/sys/netinet6/dest6.c @@ -95,7 +95,7 @@ dest6_input(struct mbuf **mp, int *offp, int proto) opt = (u_int8_t *)dstopts + sizeof(struct ip6_dest); /* search header for all options. */ - for (optlen = 0; dstoptlen > 0; dstoptlen -= optlen, opt += optlen) { + for (; dstoptlen > 0; dstoptlen -= optlen, opt += optlen) { if (*opt != IP6OPT_PAD1 && (dstoptlen < IP6OPT_MINLEN || *(opt + 1) + 2 > dstoptlen)) { IP6STAT_INC(ip6s_toosmall); diff --git a/freebsd/sys/netinet6/frag6.c b/freebsd/sys/netinet6/frag6.c index 5b405ebb..70103fe3 100644 --- a/freebsd/sys/netinet6/frag6.c +++ b/freebsd/sys/netinet6/frag6.c @@ -580,10 +580,8 @@ insert: /* * Store NXT to the original. */ - { - char *prvnxtp = ip6_get_prevhdr(m, offset); /* XXX */ - *prvnxtp = nxt; - } + m_copyback(m, ip6_get_prevhdr(m, offset), sizeof(uint8_t), + (caddr_t)&nxt); frag6_remque(q6); V_frag6_nfrags -= q6->ip6q_nfrag; diff --git a/freebsd/sys/netinet6/icmp6.c b/freebsd/sys/netinet6/icmp6.c index 38f14461..77876599 100644 --- a/freebsd/sys/netinet6/icmp6.c +++ b/freebsd/sys/netinet6/icmp6.c @@ -596,7 +596,6 @@ icmp6_input(struct mbuf **mp, int *offp, int proto) n->m_pkthdr.len = n0len + (noff - off); n->m_next = n0; } else { - nip6 = mtod(n, struct ip6_hdr *); IP6_EXTHDR_GET(nicmp6, struct icmp6_hdr *, n, off, sizeof(*nicmp6)); noff = off; @@ -2319,6 +2318,14 @@ icmp6_redirect_input(struct mbuf *m, int off) goto bad; } + /* + * Embed scope zone id into next hop address, since + * fib6_lookup_nh_basic() returns address without embedded + * scope zone id. + */ + if (in6_setscope(&nh6.nh_addr, m->m_pkthdr.rcvif, NULL)) + goto freeit; + if (IN6_ARE_ADDR_EQUAL(&src6, &nh6.nh_addr) == 0) { nd6log((LOG_ERR, "ICMP6 redirect rejected; " diff --git a/freebsd/sys/netinet6/in6.c b/freebsd/sys/netinet6/in6.c index 6fc29892..a5d84d85 100644 --- a/freebsd/sys/netinet6/in6.c +++ b/freebsd/sys/netinet6/in6.c @@ -114,6 +114,14 @@ __FBSDID("$FreeBSD$"); #include <netinet6/in6_fib.h> #include <netinet6/in6_pcb.h> +/* + * struct in6_ifreq and struct ifreq must be type punnable for common members + * of ifr_ifru to allow accessors to be shared. + */ +_Static_assert(offsetof(struct in6_ifreq, ifr_ifru) == + offsetof(struct ifreq, ifr_ifru), + "struct in6_ifreq and struct ifreq are not type punnable"); + VNET_DECLARE(int, icmp6_nodeinfo_oldmcprefix); #define V_icmp6_nodeinfo_oldmcprefix VNET(icmp6_nodeinfo_oldmcprefix) @@ -480,10 +488,6 @@ in6_control(struct socket *so, u_long cmd, caddr_t data, error = EINVAL; goto out; } - /* - * XXX: should we check if ifa_dstaddr is NULL and return - * an error? - */ ifr->ifr_dstaddr = ia->ia_dstaddr; if ((error = sa6_recoverscope(&ifr->ifr_dstaddr)) != 0) goto out; @@ -1975,8 +1979,6 @@ in6_if2idlen(struct ifnet *ifp) return (64); case IFT_FDDI: /* RFC2467 */ return (64); - case IFT_ISO88025: /* RFC2470 (IPv6 over Token Ring) */ - return (64); case IFT_PPP: /* RFC2472 */ return (64); case IFT_ARCNET: /* RFC2497 */ @@ -2152,6 +2154,25 @@ in6_lltable_rtcheck(struct ifnet *ifp, return 0; } +/* + * Called by the datapath to indicate that the entry was used. + */ +static void +in6_lltable_mark_used(struct llentry *lle) +{ + + LLE_REQ_LOCK(lle); + lle->r_skip_req = 0; + + /* + * Set the hit time so the callback function + * can determine the remaining time before + * transiting to the DELAY state. + */ + lle->lle_hittime = time_uptime; + LLE_REQ_UNLOCK(lle); +} + static inline uint32_t in6_lltable_hash_dst(const struct in6_addr *dst, uint32_t hsize) { @@ -2384,6 +2405,7 @@ in6_lltattach(struct ifnet *ifp) llt->llt_fill_sa_entry = in6_lltable_fill_sa_entry; llt->llt_free_entry = in6_lltable_free_entry; llt->llt_match_prefix = in6_lltable_match_prefix; + llt->llt_mark_used = in6_lltable_mark_used; lltable_link(llt); return (llt); diff --git a/freebsd/sys/netinet6/in6_ifattach.c b/freebsd/sys/netinet6/in6_ifattach.c index 26a682ad..1ce489f5 100644 --- a/freebsd/sys/netinet6/in6_ifattach.c +++ b/freebsd/sys/netinet6/in6_ifattach.c @@ -281,7 +281,6 @@ found: case IFT_ETHER: case IFT_L2VLAN: case IFT_FDDI: - case IFT_ISO88025: case IFT_ATM: case IFT_IEEE1394: /* IEEE802/EUI64 cases - what others? */ diff --git a/freebsd/sys/netinet6/in6_pcb.c b/freebsd/sys/netinet6/in6_pcb.c index c9107260..a4bbd6a9 100644 --- a/freebsd/sys/netinet6/in6_pcb.c +++ b/freebsd/sys/netinet6/in6_pcb.c @@ -882,6 +882,7 @@ in6_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup, struct inpcbhead *head; struct inpcb *inp, *tmpinp; u_short fport = fport_arg, lport = lport_arg; + bool locked; /* * First look for an exact match. @@ -1040,18 +1041,32 @@ in6_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup, return (NULL); found: - in_pcbref(inp); - INP_GROUP_UNLOCK(pcbgroup); - if (lookupflags & INPLOOKUP_WLOCKPCB) { - INP_WLOCK(inp); - if (in_pcbrele_wlocked(inp)) - return (NULL); - } else if (lookupflags & INPLOOKUP_RLOCKPCB) { - INP_RLOCK(inp); - if (in_pcbrele_rlocked(inp)) - return (NULL); - } else + if (lookupflags & INPLOOKUP_WLOCKPCB) + locked = INP_TRY_WLOCK(inp); + else if (lookupflags & INPLOOKUP_RLOCKPCB) + locked = INP_TRY_RLOCK(inp); + else panic("%s: locking buf", __func__); + if (!locked) + in_pcbref(inp); + INP_GROUP_UNLOCK(pcbgroup); + if (!locked) { + if (lookupflags & INPLOOKUP_WLOCKPCB) { + INP_WLOCK(inp); + if (in_pcbrele_wlocked(inp)) + return (NULL); + } else { + INP_RLOCK(inp); + if (in_pcbrele_rlocked(inp)) + return (NULL); + } + } +#ifdef INVARIANTS + if (lookupflags & INPLOOKUP_WLOCKPCB) + INP_WLOCK_ASSERT(inp); + else + INP_RLOCK_ASSERT(inp); +#endif return (inp); } #endif /* PCBGROUP */ @@ -1177,23 +1192,38 @@ in6_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in6_addr *faddr, struct ifnet *ifp) { struct inpcb *inp; + bool locked; INP_HASH_RLOCK(pcbinfo); inp = in6_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport, (lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp); if (inp != NULL) { - in_pcbref(inp); - INP_HASH_RUNLOCK(pcbinfo); - if (lookupflags & INPLOOKUP_WLOCKPCB) { - INP_WLOCK(inp); - if (in_pcbrele_wlocked(inp)) - return (NULL); - } else if (lookupflags & INPLOOKUP_RLOCKPCB) { - INP_RLOCK(inp); - if (in_pcbrele_rlocked(inp)) - return (NULL); - } else + if (lookupflags & INPLOOKUP_WLOCKPCB) + locked = INP_TRY_WLOCK(inp); + else if (lookupflags & INPLOOKUP_RLOCKPCB) + locked = INP_TRY_RLOCK(inp); + else panic("%s: locking bug", __func__); + if (!locked) + in_pcbref(inp); + INP_HASH_RUNLOCK(pcbinfo); + if (!locked) { + if (lookupflags & INPLOOKUP_WLOCKPCB) { + INP_WLOCK(inp); + if (in_pcbrele_wlocked(inp)) + return (NULL); + } else { + INP_RLOCK(inp); + if (in_pcbrele_rlocked(inp)) + return (NULL); + } + } +#ifdef INVARIANTS + if (lookupflags & INPLOOKUP_WLOCKPCB) + INP_WLOCK_ASSERT(inp); + else + INP_RLOCK_ASSERT(inp); +#endif } else INP_HASH_RUNLOCK(pcbinfo); return (inp); diff --git a/freebsd/sys/netinet6/ip6_fastfwd.c b/freebsd/sys/netinet6/ip6_fastfwd.c index e11f612b..056f9315 100644 --- a/freebsd/sys/netinet6/ip6_fastfwd.c +++ b/freebsd/sys/netinet6/ip6_fastfwd.c @@ -159,7 +159,7 @@ ip6_tryforward(struct mbuf *m) */ if (!PFIL_HOOKED(&V_inet6_pfil_hook)) goto passin; - if (pfil_run_hooks(&V_inet6_pfil_hook, &m, rcvif, PFIL_IN, + if (pfil_run_hooks(&V_inet6_pfil_hook, &m, rcvif, PFIL_IN, 0, NULL) != 0 || m == NULL) goto dropin; /* @@ -203,7 +203,7 @@ passin: if (!PFIL_HOOKED(&V_inet6_pfil_hook)) goto passout; if (pfil_run_hooks(&V_inet6_pfil_hook, &m, nh.nh_ifp, PFIL_OUT, - NULL) != 0 || m == NULL) + PFIL_FWD, NULL) != 0 || m == NULL) goto dropout; /* diff --git a/freebsd/sys/netinet6/ip6_forward.c b/freebsd/sys/netinet6/ip6_forward.c index 3364dd05..80535efe 100644 --- a/freebsd/sys/netinet6/ip6_forward.c +++ b/freebsd/sys/netinet6/ip6_forward.c @@ -326,8 +326,9 @@ again2: goto pass; odst = ip6->ip6_dst; - /* Run through list of hooks for output packets. */ - error = pfil_run_hooks(&V_inet6_pfil_hook, &m, rt->rt_ifp, PFIL_OUT, NULL); + /* Run through list of hooks for forwarded packets. */ + error = pfil_run_hooks(&V_inet6_pfil_hook, &m, rt->rt_ifp, PFIL_OUT, + PFIL_FWD, NULL); if (error != 0 || m == NULL) goto freecopy; /* consumed by filter */ ip6 = mtod(m, struct ip6_hdr *); diff --git a/freebsd/sys/netinet6/ip6_input.c b/freebsd/sys/netinet6/ip6_input.c index 78d941ae..7e1007e3 100644 --- a/freebsd/sys/netinet6/ip6_input.c +++ b/freebsd/sys/netinet6/ip6_input.c @@ -763,7 +763,7 @@ ip6_input(struct mbuf *m) odst = ip6->ip6_dst; if (pfil_run_hooks(&V_inet6_pfil_hook, &m, - m->m_pkthdr.rcvif, PFIL_IN, NULL)) + m->m_pkthdr.rcvif, PFIL_IN, 0, NULL)) return; if (m == NULL) /* consumed by filter */ return; @@ -1713,49 +1713,39 @@ ip6_pullexthdr(struct mbuf *m, size_t off, int nxt) /* * Get pointer to the previous header followed by the header * currently processed. - * XXX: This function supposes that - * M includes all headers, - * the next header field and the header length field of each header - * are valid, and - * the sum of each header length equals to OFF. - * Because of these assumptions, this function must be called very - * carefully. Moreover, it will not be used in the near future when - * we develop `neater' mechanism to process extension headers. */ -char * +int ip6_get_prevhdr(const struct mbuf *m, int off) { - struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); + struct ip6_ext ip6e; + struct ip6_hdr *ip6; + int len, nlen, nxt; if (off == sizeof(struct ip6_hdr)) - return (&ip6->ip6_nxt); - else { - int len, nxt; - struct ip6_ext *ip6e = NULL; + return (offsetof(struct ip6_hdr, ip6_nxt)); + if (off < sizeof(struct ip6_hdr)) + panic("%s: off < sizeof(struct ip6_hdr)", __func__); - nxt = ip6->ip6_nxt; - len = sizeof(struct ip6_hdr); - while (len < off) { - ip6e = (struct ip6_ext *)(mtod(m, caddr_t) + len); - - switch (nxt) { - case IPPROTO_FRAGMENT: - len += sizeof(struct ip6_frag); - break; - case IPPROTO_AH: - len += (ip6e->ip6e_len + 2) << 2; - break; - default: - len += (ip6e->ip6e_len + 1) << 3; - break; - } - nxt = ip6e->ip6e_nxt; + ip6 = mtod(m, struct ip6_hdr *); + nxt = ip6->ip6_nxt; + len = sizeof(struct ip6_hdr); + nlen = 0; + while (len < off) { + m_copydata(m, len, sizeof(ip6e), (caddr_t)&ip6e); + switch (nxt) { + case IPPROTO_FRAGMENT: + nlen = sizeof(struct ip6_frag); + break; + case IPPROTO_AH: + nlen = (ip6e.ip6e_len + 2) << 2; + break; + default: + nlen = (ip6e.ip6e_len + 1) << 3; } - if (ip6e) - return (&ip6e->ip6e_nxt); - else - return NULL; + len += nlen; + nxt = ip6e.ip6e_nxt; } + return (len - nlen); } /* diff --git a/freebsd/sys/netinet6/ip6_mroute.c b/freebsd/sys/netinet6/ip6_mroute.c index 4959145d..a4a8cdf9 100644 --- a/freebsd/sys/netinet6/ip6_mroute.c +++ b/freebsd/sys/netinet6/ip6_mroute.c @@ -1859,7 +1859,7 @@ pim6_input(struct mbuf **mp, int *offp, int proto) "of the inner packet", (eip6->ip6_vfc & IPV6_VERSION)); m_freem(m); - return (IPPROTO_NONE); + return (IPPROTO_DONE); } /* verify the inner packet is destined to a mcast group */ diff --git a/freebsd/sys/netinet6/ip6_output.c b/freebsd/sys/netinet6/ip6_output.c index 8dd71077..1841829a 100644 --- a/freebsd/sys/netinet6/ip6_output.c +++ b/freebsd/sys/netinet6/ip6_output.c @@ -137,7 +137,7 @@ static int ip6_pcbopt(int, u_char *, int, struct ip6_pktopts **, struct ucred *, int); static int ip6_pcbopts(struct ip6_pktopts **, struct mbuf *, struct socket *, struct sockopt *); -static int ip6_getpcbopt(struct ip6_pktopts *, int, struct sockopt *); +static int ip6_getpcbopt(struct inpcb *, int, struct sockopt *); static int ip6_setpktopt(int, u_char *, int, struct ip6_pktopts *, struct ucred *, int, int, int); @@ -787,7 +787,7 @@ again: odst = ip6->ip6_dst; /* Run through list of hooks for output packets. */ - error = pfil_run_hooks(&V_inet6_pfil_hook, &m, ifp, PFIL_OUT, inp); + error = pfil_run_hooks(&V_inet6_pfil_hook, &m, ifp, PFIL_OUT, 0, inp); if (error != 0 || m == NULL) goto done; /* adjust pointer */ @@ -1050,7 +1050,7 @@ sendorfree: m = m0->m_nextpkt; m0->m_nextpkt = 0; m_freem(m0); - for (m0 = m; m; m = m0) { + for (; m; m = m0) { m0 = m->m_nextpkt; m->m_nextpkt = 0; if (error == 0) { @@ -1598,23 +1598,34 @@ do { \ } while (/*CONSTCOND*/ 0) #define OPTBIT(bit) (in6p->inp_flags & (bit) ? 1 : 0) -#define OPTSET2(bit, val) do { \ - INP_WLOCK(in6p); \ +#define OPTSET2_N(bit, val) do { \ if (val) \ in6p->inp_flags2 |= bit; \ else \ in6p->inp_flags2 &= ~bit; \ +} while (0) +#define OPTSET2(bit, val) do { \ + INP_WLOCK(in6p); \ + OPTSET2_N(bit, val); \ INP_WUNLOCK(in6p); \ } while (0) #define OPTBIT2(bit) (in6p->inp_flags2 & (bit) ? 1 : 0) +#define OPTSET2292_EXCLUSIVE(bit) \ +do { \ + INP_WLOCK(in6p); \ + if (OPTBIT(IN6P_RFC2292)) { \ + error = EINVAL; \ + } else { \ + if (optval) \ + in6p->inp_flags |= (bit); \ + else \ + in6p->inp_flags &= ~(bit); \ + } \ + INP_WUNLOCK(in6p); \ +} while (/*CONSTCOND*/ 0) case IPV6_RECVPKTINFO: - /* cannot mix with RFC2292 */ - if (OPTBIT(IN6P_RFC2292)) { - error = EINVAL; - break; - } - OPTSET(IN6P_PKTINFO); + OPTSET2292_EXCLUSIVE(IN6P_PKTINFO); break; case IPV6_HOPLIMIT: @@ -1635,48 +1646,23 @@ do { \ } case IPV6_RECVHOPLIMIT: - /* cannot mix with RFC2292 */ - if (OPTBIT(IN6P_RFC2292)) { - error = EINVAL; - break; - } - OPTSET(IN6P_HOPLIMIT); + OPTSET2292_EXCLUSIVE(IN6P_HOPLIMIT); break; case IPV6_RECVHOPOPTS: - /* cannot mix with RFC2292 */ - if (OPTBIT(IN6P_RFC2292)) { - error = EINVAL; - break; - } - OPTSET(IN6P_HOPOPTS); + OPTSET2292_EXCLUSIVE(IN6P_HOPOPTS); break; case IPV6_RECVDSTOPTS: - /* cannot mix with RFC2292 */ - if (OPTBIT(IN6P_RFC2292)) { - error = EINVAL; - break; - } - OPTSET(IN6P_DSTOPTS); + OPTSET2292_EXCLUSIVE(IN6P_DSTOPTS); break; case IPV6_RECVRTHDRDSTOPTS: - /* cannot mix with RFC2292 */ - if (OPTBIT(IN6P_RFC2292)) { - error = EINVAL; - break; - } - OPTSET(IN6P_RTHDRDSTOPTS); + OPTSET2292_EXCLUSIVE(IN6P_RTHDRDSTOPTS); break; case IPV6_RECVRTHDR: - /* cannot mix with RFC2292 */ - if (OPTBIT(IN6P_RFC2292)) { - error = EINVAL; - break; - } - OPTSET(IN6P_RTHDR); + OPTSET2292_EXCLUSIVE(IN6P_RTHDR); break; case IPV6_RECVPATHMTU: @@ -1719,11 +1705,7 @@ do { \ break; case IPV6_RECVTCLASS: /* cannot mix with RFC2292 XXX */ - if (OPTBIT(IN6P_RFC2292)) { - error = EINVAL; - break; - } - OPTSET(IN6P_TCLASS); + OPTSET2292_EXCLUSIVE(IN6P_TCLASS); break; case IPV6_AUTOFLOWLABEL: OPTSET(IN6P_AUTOFLOWLABEL); @@ -1743,8 +1725,10 @@ do { \ case IPV6_RSS_LISTEN_BUCKET: if ((optval >= 0) && (optval < rss_getnumbuckets())) { + INP_WLOCK(in6p); in6p->inp_rss_listen_bucket = optval; - OPTSET2(INP_RSS_BUCKET_SET, 1); + OPTSET2_N(INP_RSS_BUCKET_SET, 1); + INP_WUNLOCK(in6p); } else { error = EINVAL; } @@ -2071,6 +2055,7 @@ do { \ { u_long pmtu = 0; struct ip6_mtuinfo mtuinfo; + struct in6_addr addr; if (!(so->so_state & SS_ISCONNECTED)) return (ENOTCONN); @@ -2078,9 +2063,14 @@ do { \ * XXX: we dot not consider the case of source * routing, or optional information to specify * the outgoing interface. + * Copy faddr out of in6p to avoid holding lock + * on inp during route lookup. */ + INP_RLOCK(in6p); + bcopy(&in6p->in6p_faddr, &addr, sizeof(addr)); + INP_RUNLOCK(in6p); error = ip6_getpmtu_ctl(so->so_fibnum, - &in6p->in6p_faddr, &pmtu); + &addr, &pmtu); if (error) break; if (pmtu > IPV6_MAXPACKET) @@ -2130,8 +2120,7 @@ do { \ case IPV6_DONTFRAG: case IPV6_USE_MIN_MTU: case IPV6_PREFER_TEMPADDR: - error = ip6_getpcbopt(in6p->in6p_outputopts, - optname, sopt); + error = ip6_getpcbopt(in6p, optname, sopt); break; case IPV6_MULTICAST_IF: @@ -2306,17 +2295,50 @@ ip6_pcbopt(int optname, u_char *buf, int len, struct ip6_pktopts **pktopt, return (ip6_setpktopt(optname, buf, len, opt, cred, 1, 0, uproto)); } +#define GET_PKTOPT_VAR(field, lenexpr) do { \ + if (pktopt && pktopt->field) { \ + INP_RUNLOCK(in6p); \ + optdata = malloc(sopt->sopt_valsize, M_TEMP, M_WAITOK); \ + malloc_optdata = true; \ + INP_RLOCK(in6p); \ + if (in6p->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { \ + INP_RUNLOCK(in6p); \ + free(optdata, M_TEMP); \ + return (ECONNRESET); \ + } \ + pktopt = in6p->in6p_outputopts; \ + if (pktopt && pktopt->field) { \ + optdatalen = min(lenexpr, sopt->sopt_valsize); \ + bcopy(&pktopt->field, optdata, optdatalen); \ + } else { \ + free(optdata, M_TEMP); \ + optdata = NULL; \ + malloc_optdata = false; \ + } \ + } \ +} while(0) + +#define GET_PKTOPT_EXT_HDR(field) GET_PKTOPT_VAR(field, \ + (((struct ip6_ext *)pktopt->field)->ip6e_len + 1) << 3) + +#define GET_PKTOPT_SOCKADDR(field) GET_PKTOPT_VAR(field, \ + pktopt->field->sa_len) + static int -ip6_getpcbopt(struct ip6_pktopts *pktopt, int optname, struct sockopt *sopt) +ip6_getpcbopt(struct inpcb *in6p, int optname, struct sockopt *sopt) { void *optdata = NULL; + bool malloc_optdata = false; int optdatalen = 0; - struct ip6_ext *ip6e; int error = 0; struct in6_pktinfo null_pktinfo; int deftclass = 0, on; int defminmtu = IP6PO_MINMTU_MCASTONLY; int defpreftemp = IP6PO_TEMPADDR_SYSTEM; + struct ip6_pktopts *pktopt; + + INP_RLOCK(in6p); + pktopt = in6p->in6p_outputopts; switch (optname) { case IPV6_PKTINFO: @@ -2333,50 +2355,29 @@ ip6_getpcbopt(struct ip6_pktopts *pktopt, int optname, struct sockopt *sopt) break; case IPV6_TCLASS: if (pktopt && pktopt->ip6po_tclass >= 0) - optdata = (void *)&pktopt->ip6po_tclass; - else - optdata = (void *)&deftclass; + deftclass = pktopt->ip6po_tclass; + optdata = (void *)&deftclass; optdatalen = sizeof(int); break; case IPV6_HOPOPTS: - if (pktopt && pktopt->ip6po_hbh) { - optdata = (void *)pktopt->ip6po_hbh; - ip6e = (struct ip6_ext *)pktopt->ip6po_hbh; - optdatalen = (ip6e->ip6e_len + 1) << 3; - } + GET_PKTOPT_EXT_HDR(ip6po_hbh); break; case IPV6_RTHDR: - if (pktopt && pktopt->ip6po_rthdr) { - optdata = (void *)pktopt->ip6po_rthdr; - ip6e = (struct ip6_ext *)pktopt->ip6po_rthdr; - optdatalen = (ip6e->ip6e_len + 1) << 3; - } + GET_PKTOPT_EXT_HDR(ip6po_rthdr); break; case IPV6_RTHDRDSTOPTS: - if (pktopt && pktopt->ip6po_dest1) { - optdata = (void *)pktopt->ip6po_dest1; - ip6e = (struct ip6_ext *)pktopt->ip6po_dest1; - optdatalen = (ip6e->ip6e_len + 1) << 3; - } + GET_PKTOPT_EXT_HDR(ip6po_dest1); break; case IPV6_DSTOPTS: - if (pktopt && pktopt->ip6po_dest2) { - optdata = (void *)pktopt->ip6po_dest2; - ip6e = (struct ip6_ext *)pktopt->ip6po_dest2; - optdatalen = (ip6e->ip6e_len + 1) << 3; - } + GET_PKTOPT_EXT_HDR(ip6po_dest2); break; case IPV6_NEXTHOP: - if (pktopt && pktopt->ip6po_nexthop) { - optdata = (void *)pktopt->ip6po_nexthop; - optdatalen = pktopt->ip6po_nexthop->sa_len; - } + GET_PKTOPT_SOCKADDR(ip6po_nexthop); break; case IPV6_USE_MIN_MTU: if (pktopt) - optdata = (void *)&pktopt->ip6po_minmtu; - else - optdata = (void *)&defminmtu; + defminmtu = pktopt->ip6po_minmtu; + optdata = (void *)&defminmtu; optdatalen = sizeof(int); break; case IPV6_DONTFRAG: @@ -2389,19 +2390,22 @@ ip6_getpcbopt(struct ip6_pktopts *pktopt, int optname, struct sockopt *sopt) break; case IPV6_PREFER_TEMPADDR: if (pktopt) - optdata = (void *)&pktopt->ip6po_prefer_tempaddr; - else - optdata = (void *)&defpreftemp; + defpreftemp = pktopt->ip6po_prefer_tempaddr; + optdata = (void *)&defpreftemp; optdatalen = sizeof(int); break; default: /* should not happen */ #ifdef DIAGNOSTIC panic("ip6_getpcbopt: unexpected option\n"); #endif + INP_RUNLOCK(in6p); return (ENOPROTOOPT); } + INP_RUNLOCK(in6p); error = sooptcopyout(sopt, optdata, optdatalen); + if (malloc_optdata) + free(optdata, M_TEMP); return (error); } diff --git a/freebsd/sys/netinet6/ip6_var.h b/freebsd/sys/netinet6/ip6_var.h index 5c8997ca..42c235d1 100644 --- a/freebsd/sys/netinet6/ip6_var.h +++ b/freebsd/sys/netinet6/ip6_var.h @@ -364,7 +364,7 @@ void ip6_direct_input(struct mbuf *); void ip6_freepcbopts(struct ip6_pktopts *); int ip6_unknown_opt(u_int8_t *, struct mbuf *, int); -char * ip6_get_prevhdr(const struct mbuf *, int); +int ip6_get_prevhdr(const struct mbuf *, int); int ip6_nexthdr(const struct mbuf *, int, int, int *); int ip6_lasthdr(const struct mbuf *, int, int, int *); diff --git a/freebsd/sys/netinet6/nd6.c b/freebsd/sys/netinet6/nd6.c index 6235b808..a00d5421 100644 --- a/freebsd/sys/netinet6/nd6.c +++ b/freebsd/sys/netinet6/nd6.c @@ -63,7 +63,6 @@ __FBSDID("$FreeBSD$"); #include <net/if_arc.h> #include <net/if_dl.h> #include <net/if_types.h> -#include <net/iso88025.h> #include <net/fddi.h> #include <net/route.h> #include <net/vnet.h> @@ -346,9 +345,6 @@ nd6_setmtu0(struct ifnet *ifp, struct nd_ifinfo *ndi) case IFT_FDDI: ndi->maxmtu = MIN(FDDIIPMTU, ifp->if_mtu); /* RFC2467 */ break; - case IFT_ISO88025: - ndi->maxmtu = MIN(ISO88025_MAX_MTU, ifp->if_mtu); - break; default: ndi->maxmtu = ifp->if_mtu; break; @@ -2281,7 +2277,6 @@ nd6_resolve(struct ifnet *ifp, int is_gw, struct mbuf *m, case IFT_FDDI: case IFT_L2VLAN: case IFT_BRIDGE: - case IFT_ISO88025: ETHER_MAP_IPV6_MULTICAST(&dst6->sin6_addr, desten); return (0); diff --git a/freebsd/sys/netinet6/nd6_nbr.c b/freebsd/sys/netinet6/nd6_nbr.c index 0c875324..e5b37877 100644 --- a/freebsd/sys/netinet6/nd6_nbr.c +++ b/freebsd/sys/netinet6/nd6_nbr.c @@ -1099,7 +1099,6 @@ nd6_ifptomac(struct ifnet *ifp) case IFT_L2VLAN: case IFT_INFINIBAND: case IFT_BRIDGE: - case IFT_ISO88025: return IF_LLADDR(ifp); default: return NULL; @@ -1122,6 +1121,7 @@ struct dadq { #define ND_OPT_NONCE_LEN32 \ ((ND_OPT_NONCE_LEN + sizeof(uint32_t) - 1)/sizeof(uint32_t)) uint32_t dad_nonce[ND_OPT_NONCE_LEN32]; + bool dad_ondadq; /* on dadq? Protected by DADQ_WLOCK. */ }; static VNET_DEFINE(TAILQ_HEAD(, dadq), dadq); @@ -1140,6 +1140,7 @@ nd6_dad_add(struct dadq *dp) DADQ_WLOCK(); TAILQ_INSERT_TAIL(&V_dadq, dp, dad_list); + dp->dad_ondadq = true; DADQ_WUNLOCK(); } @@ -1148,9 +1149,17 @@ nd6_dad_del(struct dadq *dp) { DADQ_WLOCK(); - TAILQ_REMOVE(&V_dadq, dp, dad_list); - DADQ_WUNLOCK(); - nd6_dad_rele(dp); + if (dp->dad_ondadq) { + /* + * Remove dp from the dadq and release the dadq's + * reference. + */ + TAILQ_REMOVE(&V_dadq, dp, dad_list); + dp->dad_ondadq = false; + DADQ_WUNLOCK(); + nd6_dad_rele(dp); + } else + DADQ_WUNLOCK(); } static struct dadq * @@ -1283,6 +1292,8 @@ nd6_dad_start(struct ifaddr *ifa, int delay) dp->dad_ns_icount = dp->dad_na_icount = 0; dp->dad_ns_ocount = dp->dad_ns_tcount = 0; dp->dad_ns_lcount = dp->dad_loopbackprobe = 0; + + /* Add this to the dadq and add a reference for the dadq. */ refcount_init(&dp->dad_refcnt, 1); nd6_dad_add(dp); nd6_dad_starttimer(dp, delay, 0); @@ -1303,17 +1314,9 @@ nd6_dad_stop(struct ifaddr *ifa) } nd6_dad_stoptimer(dp); - - /* - * The DAD queue entry may have been removed by nd6_dad_timer() while - * we were waiting for it to stop, so re-do the lookup. - */ - nd6_dad_rele(dp); - dp = nd6_dad_find(ifa, NULL); - if (dp == NULL) - return; - nd6_dad_del(dp); + + /* Release this function's reference, acquired by nd6_dad_find(). */ nd6_dad_rele(dp); } diff --git a/freebsd/sys/netinet6/nd6_rtr.c b/freebsd/sys/netinet6/nd6_rtr.c index 2affacbf..642faa1a 100644 --- a/freebsd/sys/netinet6/nd6_rtr.c +++ b/freebsd/sys/netinet6/nd6_rtr.c @@ -410,8 +410,11 @@ nd6_ra_input(struct mbuf *m, int off, int icmp6len) int change = (ndi->linkmtu != mtu); ndi->linkmtu = mtu; - if (change) /* in6_maxmtu may change */ + if (change) { + /* in6_maxmtu may change */ in6_setmaxmtu(); + rt_updatemtu(ifp); + } } else { nd6log((LOG_INFO, "nd6_ra_input: bogus mtu " "mtu=%lu sent from %s; " diff --git a/freebsd/sys/netinet6/raw_ip6.c b/freebsd/sys/netinet6/raw_ip6.c index b68077ef..a4843380 100644 --- a/freebsd/sys/netinet6/raw_ip6.c +++ b/freebsd/sys/netinet6/raw_ip6.c @@ -327,12 +327,10 @@ rip6_input(struct mbuf **mp, int *offp, int proto) RIP6STAT_INC(rip6s_nosockmcast); if (proto == IPPROTO_NONE) m_freem(m); - else { - char *prvnxtp = ip6_get_prevhdr(m, *offp); /* XXX */ + else icmp6_error(m, ICMP6_PARAM_PROB, ICMP6_PARAMPROB_NEXTHEADER, - prvnxtp - mtod(m, char *)); - } + ip6_get_prevhdr(m, *offp)); IP6STAT_DEC(ip6s_delivered); } return (IPPROTO_DONE); diff --git a/freebsd/sys/netinet6/udp6_usrreq.c b/freebsd/sys/netinet6/udp6_usrreq.c index 98d097f7..c2b32eb1 100644 --- a/freebsd/sys/netinet6/udp6_usrreq.c +++ b/freebsd/sys/netinet6/udp6_usrreq.c @@ -222,7 +222,6 @@ udp6_input(struct mbuf **mp, int *offp, int proto) uint8_t nxt; ifp = m->m_pkthdr.rcvif; - ip6 = mtod(m, struct ip6_hdr *); #ifndef PULLDOWN_TEST IP6_EXTHDR_CHECK(m, off, sizeof(struct udphdr), IPPROTO_DONE); @@ -232,6 +231,7 @@ udp6_input(struct mbuf **mp, int *offp, int proto) IP6_EXTHDR_GET(uh, struct udphdr *, m, off, sizeof(*uh)); if (!uh) return (IPPROTO_DONE); + ip6 = mtod(m, struct ip6_hdr *); #endif UDPSTAT_INC(udps_ipackets); diff --git a/freebsd/sys/netipsec/ipsec.c b/freebsd/sys/netipsec/ipsec.c index 4d75b51b..24a6df5b 100644 --- a/freebsd/sys/netipsec/ipsec.c +++ b/freebsd/sys/netipsec/ipsec.c @@ -112,7 +112,6 @@ VNET_PCPUSTAT_SYSINIT(ipsec4stat); VNET_PCPUSTAT_SYSUNINIT(ipsec4stat); #endif /* VIMAGE */ -VNET_DEFINE(int, ip4_ah_offsetmask) = 0; /* maybe IP_DF? */ /* DF bit on encap. 0: clear 1: set 2: copy */ VNET_DEFINE(int, ip4_ipsec_dfbit) = 0; VNET_DEFINE(int, ip4_esp_trans_deflev) = IPSEC_LEVEL_USE; @@ -121,7 +120,6 @@ VNET_DEFINE(int, ip4_ah_trans_deflev) = IPSEC_LEVEL_USE; VNET_DEFINE(int, ip4_ah_net_deflev) = IPSEC_LEVEL_USE; /* ECN ignore(-1)/forbidden(0)/allowed(1) */ VNET_DEFINE(int, ip4_ipsec_ecn) = 0; -VNET_DEFINE(int, ip4_esp_randpad) = -1; static VNET_DEFINE(int, ip4_filtertunnel) = 0; #define V_ip4_filtertunnel VNET(ip4_filtertunnel) @@ -196,9 +194,6 @@ SYSCTL_INT(_net_inet_ipsec, IPSECCTL_DEF_AH_NETLEV, ah_net_deflev, SYSCTL_INT(_net_inet_ipsec, IPSECCTL_AH_CLEARTOS, ah_cleartos, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ah_cleartos), 0, "If set, clear type-of-service field when doing AH computation."); -SYSCTL_INT(_net_inet_ipsec, IPSECCTL_AH_OFFSETMASK, ah_offsetmask, - CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ip4_ah_offsetmask), 0, - "If not set, clear offset field mask when doing AH computation."); SYSCTL_INT(_net_inet_ipsec, IPSECCTL_DFBIT, dfbit, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ip4_ipsec_dfbit), 0, "Do not fragment bit on encap."); diff --git a/freebsd/sys/netipsec/ipsec.h b/freebsd/sys/netipsec/ipsec.h index a1e27bbf..ac9361a8 100644 --- a/freebsd/sys/netipsec/ipsec.h +++ b/freebsd/sys/netipsec/ipsec.h @@ -280,10 +280,8 @@ VNET_DECLARE(int, ip4_esp_trans_deflev); VNET_DECLARE(int, ip4_esp_net_deflev); VNET_DECLARE(int, ip4_ah_trans_deflev); VNET_DECLARE(int, ip4_ah_net_deflev); -VNET_DECLARE(int, ip4_ah_offsetmask); VNET_DECLARE(int, ip4_ipsec_dfbit); VNET_DECLARE(int, ip4_ipsec_ecn); -VNET_DECLARE(int, ip4_esp_randpad); VNET_DECLARE(int, crypto_support); VNET_DECLARE(int, async_crypto); VNET_DECLARE(int, natt_cksum_policy); @@ -294,10 +292,8 @@ VNET_DECLARE(int, natt_cksum_policy); #define V_ip4_esp_net_deflev VNET(ip4_esp_net_deflev) #define V_ip4_ah_trans_deflev VNET(ip4_ah_trans_deflev) #define V_ip4_ah_net_deflev VNET(ip4_ah_net_deflev) -#define V_ip4_ah_offsetmask VNET(ip4_ah_offsetmask) #define V_ip4_ipsec_dfbit VNET(ip4_ipsec_dfbit) #define V_ip4_ipsec_ecn VNET(ip4_ipsec_ecn) -#define V_ip4_esp_randpad VNET(ip4_esp_randpad) #define V_crypto_support VNET(crypto_support) #define V_async_crypto VNET(async_crypto) #define V_natt_cksum_policy VNET(natt_cksum_policy) diff --git a/freebsd/sys/netipsec/keysock.c b/freebsd/sys/netipsec/keysock.c index 7ecd50b8..170335bc 100644 --- a/freebsd/sys/netipsec/keysock.c +++ b/freebsd/sys/netipsec/keysock.c @@ -174,104 +174,20 @@ key_sendup0(struct rawcb *rp, struct mbuf *m, int promisc) return error; } -/* XXX this interface should be obsoleted. */ -int -key_sendup(struct socket *so, struct sadb_msg *msg, u_int len, int target) -{ - struct mbuf *m, *n, *mprev; - int tlen; - - /* sanity check */ - if (so == NULL || msg == NULL) - panic("%s: NULL pointer was passed.\n", __func__); - - KEYDBG(KEY_DUMP, - printf("%s: \n", __func__); - kdebug_sadb(msg)); - - /* - * we increment statistics here, just in case we have ENOBUFS - * in this function. - */ - PFKEYSTAT_INC(in_total); - PFKEYSTAT_ADD(in_bytes, len); - PFKEYSTAT_INC(in_msgtype[msg->sadb_msg_type]); - - /* - * Get mbuf chain whenever possible (not clusters), - * to save socket buffer. We'll be generating many SADB_ACQUIRE - * messages to listening key sockets. If we simply allocate clusters, - * sbappendaddr() will raise ENOBUFS due to too little sbspace(). - * sbspace() computes # of actual data bytes AND mbuf region. - * - * TODO: SADB_ACQUIRE filters should be implemented. - */ - tlen = len; - m = mprev = NULL; - while (tlen > 0) { - if (tlen == len) { - MGETHDR(n, M_NOWAIT, MT_DATA); - if (n == NULL) { - PFKEYSTAT_INC(in_nomem); - return ENOBUFS; - } - n->m_len = MHLEN; - } else { - MGET(n, M_NOWAIT, MT_DATA); - if (n == NULL) { - PFKEYSTAT_INC(in_nomem); - return ENOBUFS; - } - n->m_len = MLEN; - } - if (tlen >= MCLBYTES) { /*XXX better threshold? */ - if (!(MCLGET(n, M_NOWAIT))) { - m_free(n); - m_freem(m); - PFKEYSTAT_INC(in_nomem); - return ENOBUFS; - } - n->m_len = MCLBYTES; - } - - if (tlen < n->m_len) - n->m_len = tlen; - n->m_next = NULL; - if (m == NULL) - m = mprev = n; - else { - mprev->m_next = n; - mprev = n; - } - tlen -= n->m_len; - n = NULL; - } - m->m_pkthdr.len = len; - m->m_pkthdr.rcvif = NULL; - m_copyback(m, 0, len, (caddr_t)msg); - - /* avoid duplicated statistics */ - PFKEYSTAT_ADD(in_total, -1); - PFKEYSTAT_ADD(in_bytes, -len); - PFKEYSTAT_ADD(in_msgtype[msg->sadb_msg_type], -1); - - return key_sendup_mbuf(so, m, target); -} - /* so can be NULL if target != KEY_SENDUP_ONE */ int key_sendup_mbuf(struct socket *so, struct mbuf *m, int target) { struct mbuf *n; struct keycb *kp; - int sendup; struct rawcb *rp; int error = 0; - if (m == NULL) - panic("key_sendup_mbuf: NULL pointer was passed.\n"); - if (so == NULL && target == KEY_SENDUP_ONE) - panic("%s: NULL pointer was passed.\n", __func__); + KASSERT(m != NULL, ("NULL mbuf pointer was passed.")); + KASSERT(so != NULL || target != KEY_SENDUP_ONE, + ("NULL socket pointer was passed.")); + KASSERT(target == KEY_SENDUP_ONE || target == KEY_SENDUP_ALL || + target == KEY_SENDUP_REGISTERED, ("Wrong target %d", target)); PFKEYSTAT_INC(in_total); PFKEYSTAT_ADD(in_bytes, m->m_pkthdr.len); @@ -288,6 +204,11 @@ key_sendup_mbuf(struct socket *so, struct mbuf *m, int target) PFKEYSTAT_INC(in_msgtype[msg->sadb_msg_type]); } mtx_lock(&rawcb_mtx); + if (V_key_cb.any_count == 0) { + mtx_unlock(&rawcb_mtx); + m_freem(m); + return (0); + } LIST_FOREACH(rp, &V_rawcb_list, list) { if (rp->rcb_proto.sp_family != PF_KEY) @@ -297,69 +218,50 @@ key_sendup_mbuf(struct socket *so, struct mbuf *m, int target) continue; } - kp = (struct keycb *)rp; - /* * If you are in promiscuous mode, and when you get broadcasted * reply, you'll get two PF_KEY messages. * (based on pf_key@inner.net message on 14 Oct 1998) */ - if (((struct keycb *)rp)->kp_promisc) { - if ((n = m_copym(m, 0, M_COPYALL, M_NOWAIT)) != NULL) { - (void)key_sendup0(rp, n, 1); - n = NULL; - } + kp = (struct keycb *)rp; + if (kp->kp_promisc) { + n = m_copym(m, 0, M_COPYALL, M_NOWAIT); + if (n != NULL) + key_sendup0(rp, n, 1); + else + PFKEYSTAT_INC(in_nomem); } /* the exact target will be processed later */ if (so && sotorawcb(so) == rp) continue; - sendup = 0; - switch (target) { - case KEY_SENDUP_ONE: - /* the statement has no effect */ - if (so && sotorawcb(so) == rp) - sendup++; - break; - case KEY_SENDUP_ALL: - sendup++; - break; - case KEY_SENDUP_REGISTERED: - if (kp->kp_registered) - sendup++; - break; - } - PFKEYSTAT_INC(in_msgtarget[target]); - - if (!sendup) + if (target == KEY_SENDUP_ONE || ( + target == KEY_SENDUP_REGISTERED && kp->kp_registered == 0)) continue; - if ((n = m_copym(m, 0, M_COPYALL, M_NOWAIT)) == NULL) { - m_freem(m); + /* KEY_SENDUP_ALL + KEY_SENDUP_REGISTERED */ + n = m_copym(m, 0, M_COPYALL, M_NOWAIT); + if (n == NULL) { PFKEYSTAT_INC(in_nomem); - mtx_unlock(&rawcb_mtx); - return ENOBUFS; - } - - if ((error = key_sendup0(rp, n, 0)) != 0) { - m_freem(m); - mtx_unlock(&rawcb_mtx); - return error; + /* Try send to another socket */ + continue; } - n = NULL; + if (key_sendup0(rp, n, 0) == 0) + PFKEYSTAT_INC(in_msgtarget[target]); } - if (so) { + if (so) { /* KEY_SENDUP_ONE */ error = key_sendup0(sotorawcb(so), m, 0); - m = NULL; + if (error == 0) + PFKEYSTAT_INC(in_msgtarget[KEY_SENDUP_ONE]); } else { error = 0; m_freem(m); } mtx_unlock(&rawcb_mtx); - return error; + return (error); } /* diff --git a/freebsd/sys/netipsec/keysock.h b/freebsd/sys/netipsec/keysock.h index 30b68da6..484c6536 100644 --- a/freebsd/sys/netipsec/keysock.h +++ b/freebsd/sys/netipsec/keysock.h @@ -78,12 +78,8 @@ VNET_PCPUSTAT_DECLARE(struct pfkeystat, pfkeystat); VNET_PCPUSTAT_ADD(struct pfkeystat, pfkeystat, name, (val)) #define PFKEYSTAT_INC(name) PFKEYSTAT_ADD(name, 1) -extern int key_output(struct mbuf *m, struct socket *so, ...); -extern int key_usrreq(struct socket *, int, struct mbuf *, - struct mbuf *, struct mbuf *); - -extern int key_sendup(struct socket *, struct sadb_msg *, u_int, int); -extern int key_sendup_mbuf(struct socket *, struct mbuf *, int); +int key_output(struct mbuf *m, struct socket *so, ...); +int key_sendup_mbuf(struct socket *, struct mbuf *, int); #endif /* _KERNEL */ #endif /*_NETIPSEC_KEYSOCK_H_*/ diff --git a/freebsd/sys/netipsec/xform.h b/freebsd/sys/netipsec/xform.h index 8e6f8bdb..2720f72a 100644 --- a/freebsd/sys/netipsec/xform.h +++ b/freebsd/sys/netipsec/xform.h @@ -76,6 +76,7 @@ struct xform_data { int protoff; /* current protocol offset */ int skip; /* data offset */ uint8_t nxt; /* next protocol, e.g. IPV4 */ + struct vnet *vnet; }; #define XF_IP4 1 /* unused */ diff --git a/freebsd/sys/netipsec/xform_ah.c b/freebsd/sys/netipsec/xform_ah.c index 9125ba40..13999f41 100644 --- a/freebsd/sys/netipsec/xform_ah.c +++ b/freebsd/sys/netipsec/xform_ah.c @@ -584,6 +584,16 @@ ah_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff) error = EACCES; goto bad; } + if (skip + authsize + rplen > m->m_pkthdr.len) { + DPRINTF(("%s: bad mbuf length %u (expecting %lu)" + " for packet in SA %s/%08lx\n", __func__, + m->m_pkthdr.len, (u_long) (skip + authsize + rplen), + ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), + (u_long) ntohl(sav->spi))); + AHSTAT_INC(ahs_badauthl); + error = EACCES; + goto bad; + } AHSTAT_ADD(ahs_ibytes, m->m_pkthdr.len - skip - hl); /* Get crypto descriptors. */ @@ -628,6 +638,9 @@ ah_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff) /* Zeroize the authenticator on the packet. */ m_copyback(m, skip + rplen, authsize, ipseczeroes); + /* Save ah_nxt, since ah pointer can become invalid after "massage" */ + hl = ah->ah_nxt; + /* "Massage" the packet headers for crypto processing. */ error = ah_massage_headers(&m, sav->sah->saidx.dst.sa.sa_family, skip, ahx->type, 0); @@ -652,10 +665,11 @@ ah_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff) /* These are passed as-is to the callback. */ xd->sav = sav; - xd->nxt = ah->ah_nxt; + xd->nxt = hl; xd->protoff = protoff; xd->skip = skip; xd->cryptoid = cryptoid; + xd->vnet = curvnet; return (crypto_dispatch(crp)); bad: m_freem(m); @@ -682,6 +696,7 @@ ah_input_cb(struct cryptop *crp) m = (struct mbuf *) crp->crp_buf; xd = (struct xform_data *) crp->crp_opaque; + CURVNET_SET(xd->vnet); sav = xd->sav; skip = xd->skip; nxt = xd->nxt; @@ -699,6 +714,7 @@ ah_input_cb(struct cryptop *crp) if (ipsec_updateid(sav, &crp->crp_sid, &cryptoid) != 0) crypto_freesession(cryptoid); xd->cryptoid = crp->crp_sid; + CURVNET_RESTORE(); return (crypto_dispatch(crp)); } AHSTAT_INC(ahs_noxform); @@ -794,8 +810,10 @@ ah_input_cb(struct cryptop *crp) panic("%s: Unexpected address family: %d saidx=%p", __func__, saidx->dst.sa.sa_family, saidx); } + CURVNET_RESTORE(); return error; bad: + CURVNET_RESTORE(); if (sav) key_freesav(&sav); if (m != NULL) @@ -1029,6 +1047,7 @@ ah_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav, xd->skip = skip; xd->idx = idx; xd->cryptoid = cryptoid; + xd->vnet = curvnet; return crypto_dispatch(crp); bad: @@ -1056,6 +1075,7 @@ ah_output_cb(struct cryptop *crp) m = (struct mbuf *) crp->crp_buf; xd = (struct xform_data *) crp->crp_opaque; + CURVNET_SET(xd->vnet); sp = xd->sp; sav = xd->sav; skip = xd->skip; @@ -1070,6 +1090,7 @@ ah_output_cb(struct cryptop *crp) if (ipsec_updateid(sav, &crp->crp_sid, &cryptoid) != 0) crypto_freesession(cryptoid); xd->cryptoid = crp->crp_sid; + CURVNET_RESTORE(); return (crypto_dispatch(crp)); } AHSTAT_INC(ahs_noxform); @@ -1111,8 +1132,10 @@ ah_output_cb(struct cryptop *crp) /* NB: m is reclaimed by ipsec_process_done. */ error = ipsec_process_done(m, sp, sav, idx); + CURVNET_RESTORE(); return (error); bad: + CURVNET_RESTORE(); free(xd, M_XDATA); crypto_freereq(crp); key_freesav(&sav); diff --git a/freebsd/sys/netipsec/xform_esp.c b/freebsd/sys/netipsec/xform_esp.c index f26b8ae7..49b08ba6 100644 --- a/freebsd/sys/netipsec/xform_esp.c +++ b/freebsd/sys/netipsec/xform_esp.c @@ -399,6 +399,7 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff) xd->protoff = protoff; xd->skip = skip; xd->cryptoid = cryptoid; + xd->vnet = curvnet; /* Decryption descriptor */ IPSEC_ASSERT(crde != NULL, ("null esp crypto descriptor")); @@ -457,6 +458,7 @@ esp_input_cb(struct cryptop *crp) m = (struct mbuf *) crp->crp_buf; xd = (struct xform_data *) crp->crp_opaque; + CURVNET_SET(xd->vnet); sav = xd->sav; skip = xd->skip; protoff = xd->protoff; @@ -471,6 +473,7 @@ esp_input_cb(struct cryptop *crp) if (ipsec_updateid(sav, &crp->crp_sid, &cryptoid) != 0) crypto_freesession(cryptoid); xd->cryptoid = crp->crp_sid; + CURVNET_RESTORE(); return (crypto_dispatch(crp)); } ESPSTAT_INC(esps_noxform); @@ -605,8 +608,10 @@ esp_input_cb(struct cryptop *crp) panic("%s: Unexpected address family: %d saidx=%p", __func__, saidx->dst.sa.sa_family, saidx); } + CURVNET_RESTORE(); return error; bad: + CURVNET_RESTORE(); if (sav != NULL) key_freesav(&sav); if (m != NULL) @@ -839,6 +844,7 @@ esp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav, xd->sav = sav; xd->idx = idx; xd->cryptoid = cryptoid; + xd->vnet = curvnet; /* Crypto operation descriptor. */ crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */ @@ -884,6 +890,7 @@ esp_output_cb(struct cryptop *crp) int error; xd = (struct xform_data *) crp->crp_opaque; + CURVNET_SET(xd->vnet); m = (struct mbuf *) crp->crp_buf; sp = xd->sp; sav = xd->sav; @@ -897,6 +904,7 @@ esp_output_cb(struct cryptop *crp) if (ipsec_updateid(sav, &crp->crp_sid, &cryptoid) != 0) crypto_freesession(cryptoid); xd->cryptoid = crp->crp_sid; + CURVNET_RESTORE(); return (crypto_dispatch(crp)); } ESPSTAT_INC(esps_noxform); @@ -942,8 +950,10 @@ esp_output_cb(struct cryptop *crp) /* NB: m is reclaimed by ipsec_process_done. */ error = ipsec_process_done(m, sp, sav, idx); + CURVNET_RESTORE(); return (error); bad: + CURVNET_RESTORE(); free(xd, M_XDATA); crypto_freereq(crp); key_freesav(&sav); diff --git a/freebsd/sys/netipsec/xform_ipcomp.c b/freebsd/sys/netipsec/xform_ipcomp.c index 956383d5..b3fdee49 100644 --- a/freebsd/sys/netipsec/xform_ipcomp.c +++ b/freebsd/sys/netipsec/xform_ipcomp.c @@ -257,6 +257,7 @@ ipcomp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff) xd->sav = sav; xd->protoff = protoff; xd->skip = skip; + xd->vnet = curvnet; SECASVAR_LOCK(sav); crp->crp_sid = xd->cryptoid = sav->tdb_cryptoid; @@ -288,6 +289,7 @@ ipcomp_input_cb(struct cryptop *crp) m = (struct mbuf *) crp->crp_buf; xd = (struct xform_data *) crp->crp_opaque; + CURVNET_SET(xd->vnet); sav = xd->sav; skip = xd->skip; protoff = xd->protoff; @@ -304,6 +306,7 @@ ipcomp_input_cb(struct cryptop *crp) if (ipsec_updateid(sav, &crp->crp_sid, &cryptoid) != 0) crypto_freesession(cryptoid); xd->cryptoid = crp->crp_sid; + CURVNET_RESTORE(); return (crypto_dispatch(crp)); } IPCOMPSTAT_INC(ipcomps_noxform); @@ -368,8 +371,10 @@ ipcomp_input_cb(struct cryptop *crp) panic("%s: Unexpected address family: %d saidx=%p", __func__, saidx->dst.sa.sa_family, saidx); } + CURVNET_RESTORE(); return error; bad: + CURVNET_RESTORE(); if (sav != NULL) key_freesav(&sav); if (m != NULL) @@ -495,6 +500,7 @@ ipcomp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav, xd->idx = idx; xd->skip = skip; xd->protoff = protoff; + xd->vnet = curvnet; /* Crypto operation descriptor */ crp->crp_ilen = m->m_pkthdr.len; /* Total input length */ @@ -533,6 +539,7 @@ ipcomp_output_cb(struct cryptop *crp) m = (struct mbuf *) crp->crp_buf; xd = (struct xform_data *) crp->crp_opaque; + CURVNET_SET(xd->vnet); idx = xd->idx; sp = xd->sp; sav = xd->sav; @@ -547,6 +554,7 @@ ipcomp_output_cb(struct cryptop *crp) if (ipsec_updateid(sav, &crp->crp_sid, &cryptoid) != 0) crypto_freesession(cryptoid); xd->cryptoid = crp->crp_sid; + CURVNET_RESTORE(); return (crypto_dispatch(crp)); } IPCOMPSTAT_INC(ipcomps_noxform); @@ -642,10 +650,12 @@ ipcomp_output_cb(struct cryptop *crp) /* NB: m is reclaimed by ipsec_process_done. */ error = ipsec_process_done(m, sp, sav, idx); + CURVNET_RESTORE(); return (error); bad: if (m) m_freem(m); + CURVNET_RESTORE(); free(xd, M_XDATA); crypto_freereq(crp); key_freesav(&sav); diff --git a/freebsd/sys/netpfil/ipfw/ip_fw_private.h b/freebsd/sys/netpfil/ipfw/ip_fw_private.h index b6471a02..c389e01a 100644 --- a/freebsd/sys/netpfil/ipfw/ip_fw_private.h +++ b/freebsd/sys/netpfil/ipfw/ip_fw_private.h @@ -1,4 +1,6 @@ /*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * * Copyright (c) 2002-2009 Luigi Rizzo, Universita` di Pisa * * Redistribution and use in source and binary forms, with or without @@ -182,24 +184,48 @@ enum { /* result for matching dynamic rules */ struct ip_fw_chain; struct sockopt_data; int ipfw_is_dyn_rule(struct ip_fw *rule); -void ipfw_expire_dyn_rules(struct ip_fw_chain *, ipfw_range_tlv *); -void ipfw_dyn_unlock(ipfw_dyn_rule *q); +void ipfw_expire_dyn_states(struct ip_fw_chain *, ipfw_range_tlv *); struct tcphdr; struct mbuf *ipfw_send_pkt(struct mbuf *, struct ipfw_flow_id *, u_int32_t, u_int32_t, int); -int ipfw_install_state(struct ip_fw_chain *chain, struct ip_fw *rule, - ipfw_insn_limit *cmd, struct ip_fw_args *args, uint32_t tablearg); -ipfw_dyn_rule *ipfw_lookup_dyn_rule(struct ipfw_flow_id *pkt, - int *match_direction, struct tcphdr *tcp, uint16_t kidx); -void ipfw_remove_dyn_children(struct ip_fw *rule); +/* + * Macro to determine that we need to do or redo dynamic state lookup. + * direction == MATCH_UNKNOWN means that this is first lookup, then we need + * to do lookup. + * Otherwise check the state name, if previous lookup was for "any" name, + * this means there is no state with specific name. Thus no need to do + * lookup. If previous name was not "any", redo lookup for specific name. + */ +#define DYN_LOOKUP_NEEDED(p, cmd) \ + ((p)->direction == MATCH_UNKNOWN || \ + ((p)->kidx != 0 && (p)->kidx != (cmd)->arg1)) +#define DYN_INFO_INIT(p) do { \ + (p)->direction = MATCH_UNKNOWN; \ + (p)->kidx = 0; \ +} while (0) +struct ipfw_dyn_info { + uint16_t direction; /* match direction */ + uint16_t kidx; /* state name kidx */ + uint32_t hashval; /* hash value */ + uint32_t version; /* bucket version */ + uint32_t f_pos; +}; +int ipfw_dyn_install_state(struct ip_fw_chain *chain, struct ip_fw *rule, + const ipfw_insn_limit *cmd, const struct ip_fw_args *args, + const void *ulp, int pktlen, struct ipfw_dyn_info *info, + uint32_t tablearg); +struct ip_fw *ipfw_dyn_lookup_state(const struct ip_fw_args *args, + const void *ulp, int pktlen, const ipfw_insn *cmd, + struct ipfw_dyn_info *info); + void ipfw_get_dynamic(struct ip_fw_chain *chain, char **bp, const char *ep); int ipfw_dump_states(struct ip_fw_chain *chain, struct sockopt_data *sd); void ipfw_dyn_init(struct ip_fw_chain *); /* per-vnet initialization */ void ipfw_dyn_uninit(int); /* per-vnet deinitialization */ int ipfw_dyn_len(void); -int ipfw_dyn_get_count(void); +uint32_t ipfw_dyn_get_count(void); /* common variables */ VNET_DECLARE(int, fw_one_pass); @@ -625,6 +651,8 @@ void ipfw_destroy_skipto_cache(struct ip_fw_chain *chain); int ipfw_find_rule(struct ip_fw_chain *chain, uint32_t key, uint32_t id); int ipfw_ctl3(struct sockopt *sopt); int ipfw_chk(struct ip_fw_args *args); +int ipfw_add_protected_rule(struct ip_fw_chain *chain, struct ip_fw *rule, + int locked); void ipfw_reap_add(struct ip_fw_chain *chain, struct ip_fw **head, struct ip_fw *rule); void ipfw_reap_rules(struct ip_fw *head); diff --git a/freebsd/sys/netpfil/pf/if_pfsync.c b/freebsd/sys/netpfil/pf/if_pfsync.c index 2343adc2..3ed1b3bb 100644 --- a/freebsd/sys/netpfil/pf/if_pfsync.c +++ b/freebsd/sys/netpfil/pf/if_pfsync.c @@ -1321,7 +1321,8 @@ pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data) pfsyncr.pfsyncr_defer = (PFSYNCF_DEFER == (sc->sc_flags & PFSYNCF_DEFER)); PFSYNC_UNLOCK(sc); - return (copyout(&pfsyncr, ifr->ifr_data, sizeof(pfsyncr))); + return (copyout(&pfsyncr, ifr_data_get_ptr(ifr), + sizeof(pfsyncr))); case SIOCSETPFSYNC: { @@ -1332,7 +1333,8 @@ pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data) if ((error = priv_check(curthread, PRIV_NETINET_PF)) != 0) return (error); - if ((error = copyin(ifr->ifr_data, &pfsyncr, sizeof(pfsyncr)))) + if ((error = copyin(ifr_data_get_ptr(ifr), &pfsyncr, + sizeof(pfsyncr)))) return (error); if (pfsyncr.pfsyncr_maxupdates > 255) diff --git a/freebsd/sys/netpfil/pf/pf.c b/freebsd/sys/netpfil/pf/pf.c index a904a0db..1aab6f49 100644 --- a/freebsd/sys/netpfil/pf/pf.c +++ b/freebsd/sys/netpfil/pf/pf.c @@ -72,6 +72,7 @@ __FBSDID("$FreeBSD$"); #include <net/radix_mpath.h> #include <net/vnet.h> +#include <net/pfil.h> #include <net/pfvar.h> #include <net/if_pflog.h> #include <net/if_pfsync.h> @@ -792,7 +793,7 @@ pf_initialize() if (pf_hashsize == 0 || !powerof2(pf_hashsize)) pf_hashsize = PF_HASHSIZ; if (pf_srchashsize == 0 || !powerof2(pf_srchashsize)) - pf_srchashsize = PF_HASHSIZ / 4; + pf_srchashsize = PF_SRCHASHSIZ; V_pf_hashseed = arc4random(); @@ -806,10 +807,25 @@ pf_initialize() V_pf_state_key_z = uma_zcreate("pf state keys", sizeof(struct pf_state_key), pf_state_key_ctor, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); - V_pf_keyhash = malloc(pf_hashsize * sizeof(struct pf_keyhash), - M_PFHASH, M_WAITOK | M_ZERO); - V_pf_idhash = malloc(pf_hashsize * sizeof(struct pf_idhash), - M_PFHASH, M_WAITOK | M_ZERO); + + V_pf_keyhash = mallocarray(pf_hashsize, sizeof(struct pf_keyhash), + M_PFHASH, M_NOWAIT | M_ZERO); + V_pf_idhash = mallocarray(pf_hashsize, sizeof(struct pf_idhash), + M_PFHASH, M_NOWAIT | M_ZERO); + if (V_pf_keyhash == NULL || V_pf_idhash == NULL) { + printf("pf: Unable to allocate memory for " + "state_hashsize %lu.\n", pf_hashsize); + + free(V_pf_keyhash, M_PFHASH); + free(V_pf_idhash, M_PFHASH); + + pf_hashsize = PF_HASHSIZ; + V_pf_keyhash = mallocarray(pf_hashsize, + sizeof(struct pf_keyhash), M_PFHASH, M_WAITOK | M_ZERO); + V_pf_idhash = mallocarray(pf_hashsize, + sizeof(struct pf_idhash), M_PFHASH, M_WAITOK | M_ZERO); + } + pf_hashmask = pf_hashsize - 1; for (i = 0, kh = V_pf_keyhash, ih = V_pf_idhash; i <= pf_hashmask; i++, kh++, ih++) { @@ -824,8 +840,18 @@ pf_initialize() V_pf_limits[PF_LIMIT_SRC_NODES].zone = V_pf_sources_z; uma_zone_set_max(V_pf_sources_z, PFSNODE_HIWAT); uma_zone_set_warning(V_pf_sources_z, "PF source nodes limit reached"); - V_pf_srchash = malloc(pf_srchashsize * sizeof(struct pf_srchash), - M_PFHASH, M_WAITOK|M_ZERO); + + V_pf_srchash = mallocarray(pf_srchashsize, + sizeof(struct pf_srchash), M_PFHASH, M_NOWAIT | M_ZERO); + if (V_pf_srchash == NULL) { + printf("pf: Unable to allocate memory for " + "source_hashsize %lu.\n", pf_srchashsize); + + pf_srchashsize = PF_SRCHASHSIZ; + V_pf_srchash = mallocarray(pf_srchashsize, + sizeof(struct pf_srchash), M_PFHASH, M_WAITOK | M_ZERO); + } + pf_srchashmask = pf_srchashsize - 1; for (i = 0, sh = V_pf_srchash; i <= pf_srchashmask; i++, sh++) mtx_init(&sh->lock, "pf_srchash", NULL, MTX_DEF); @@ -1615,7 +1641,6 @@ int pf_unlink_state(struct pf_state *s, u_int flags) { struct pf_idhash *ih = &V_pf_idhash[PF_IDHASH(s)]; - int last; if ((flags & PF_ENTER_LOCKED) == 0) PF_HASHROW_LOCK(ih); @@ -1656,8 +1681,9 @@ pf_unlink_state(struct pf_state *s, u_int flags) PF_HASHROW_UNLOCK(ih); pf_detach_state(s); - last = refcount_release(&s->refs); - KASSERT(last == 0, ("Incorrect state reference count")); + /* pf_state_insert() initialises refs to 2, so we can never release the + * last reference here, only in pf_release_state(). */ + (void)refcount_release(&s->refs); return (pf_release_state(s)); } @@ -5493,7 +5519,7 @@ pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, goto bad; if (oifp != ifp) { - if (pf_test(PF_OUT, ifp, &m0, NULL) != PF_PASS) + if (pf_test(PF_OUT, 0, ifp, &m0, NULL) != PF_PASS) goto bad; else if (m0 == NULL) goto done; @@ -5655,7 +5681,7 @@ pf_route6(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, goto bad; if (oifp != ifp) { - if (pf_test6(PF_FWD, ifp, &m0, NULL) != PF_PASS) + if (pf_test6(PF_OUT, PFIL_FWD, ifp, &m0, NULL) != PF_PASS) goto bad; else if (m0 == NULL) goto done; @@ -5845,7 +5871,7 @@ pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, sa_family_t a #ifdef INET int -pf_test(int dir, struct ifnet *ifp, struct mbuf **m0, struct inpcb *inp) +pf_test(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0, struct inpcb *inp) { struct pfi_kif *kif; u_short action, reason = 0, log = 0; @@ -6232,7 +6258,7 @@ done: #ifdef INET6 int -pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0, struct inpcb *inp) +pf_test6(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0, struct inpcb *inp) { struct pfi_kif *kif; u_short action, reason = 0, log = 0; @@ -6244,28 +6270,9 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0, struct inpcb *inp) struct pf_ruleset *ruleset = NULL; struct pf_pdesc pd; int off, terminal = 0, dirndx, rh_cnt = 0, pqid = 0; - int fwdir = dir; M_ASSERTPKTHDR(m); - /* Detect packet forwarding. - * If the input interface is different from the output interface we're - * forwarding. - * We do need to be careful about bridges. If the - * net.link.bridge.pfil_bridge sysctl is set we can be filtering on a - * bridge, so if the input interface is a bridge member and the output - * interface is its bridge or a member of the same bridge we're not - * actually forwarding but bridging. - */ - if (dir == PF_OUT && m->m_pkthdr.rcvif && ifp != m->m_pkthdr.rcvif && - (m->m_pkthdr.rcvif->if_bridge == NULL || - (m->m_pkthdr.rcvif->if_bridge != ifp->if_softc && - m->m_pkthdr.rcvif->if_bridge != ifp->if_bridge))) - fwdir = PF_FWD; - - if (dir == PF_FWD) - dir = PF_OUT; - if (!V_pf_status.running) return (PF_PASS); @@ -6643,7 +6650,7 @@ done: PF_STATE_UNLOCK(s); /* If reassembled packet passed, create new fragments. */ - if (action == PF_PASS && *m0 && fwdir == PF_FWD && + if (action == PF_PASS && *m0 && (pflags & PFIL_FWD) && (mtag = m_tag_find(m, PF_REASSEMBLED, NULL)) != NULL) action = pf_refragment6(ifp, m0, mtag); diff --git a/freebsd/sys/netpfil/pf/pf.h b/freebsd/sys/netpfil/pf/pf.h index 69472782..253e8804 100644 --- a/freebsd/sys/netpfil/pf/pf.h +++ b/freebsd/sys/netpfil/pf/pf.h @@ -45,7 +45,7 @@ #endif #endif -enum { PF_INOUT, PF_IN, PF_OUT, PF_FWD }; +enum { PF_INOUT, PF_IN, PF_OUT }; enum { PF_PASS, PF_DROP, PF_SCRUB, PF_NOSCRUB, PF_NAT, PF_NONAT, PF_BINAT, PF_NOBINAT, PF_RDR, PF_NORDR, PF_SYNPROXY_DROP, PF_DEFER }; enum { PF_RULESET_SCRUB, PF_RULESET_FILTER, PF_RULESET_NAT, diff --git a/freebsd/sys/netpfil/pf/pf_ioctl.c b/freebsd/sys/netpfil/pf/pf_ioctl.c index a1b0b5e5..9be57273 100644 --- a/freebsd/sys/netpfil/pf/pf_ioctl.c +++ b/freebsd/sys/netpfil/pf/pf_ioctl.c @@ -167,15 +167,15 @@ static void pf_tbladdr_copyout(struct pf_addr_wrap *); */ #ifdef INET static int pf_check_in(void *arg, struct mbuf **m, struct ifnet *ifp, - int dir, struct inpcb *inp); + int dir, int flags, struct inpcb *inp); static int pf_check_out(void *arg, struct mbuf **m, struct ifnet *ifp, - int dir, struct inpcb *inp); + int dir, int flags, struct inpcb *inp); #endif #ifdef INET6 static int pf_check6_in(void *arg, struct mbuf **m, struct ifnet *ifp, - int dir, struct inpcb *inp); + int dir, int flags, struct inpcb *inp); static int pf_check6_out(void *arg, struct mbuf **m, struct ifnet *ifp, - int dir, struct inpcb *inp); + int dir, int flags, struct inpcb *inp); #endif static int hook_pf(void); @@ -3661,12 +3661,12 @@ shutdown_pf(void) #ifdef INET static int -pf_check_in(void *arg, struct mbuf **m, struct ifnet *ifp, int dir, +pf_check_in(void *arg, struct mbuf **m, struct ifnet *ifp, int dir, int flags, struct inpcb *inp) { int chk; - chk = pf_test(PF_IN, ifp, m, inp); + chk = pf_test(PF_IN, flags, ifp, m, inp); if (chk && *m) { m_freem(*m); *m = NULL; @@ -3678,12 +3678,12 @@ pf_check_in(void *arg, struct mbuf **m, struct ifnet *ifp, int dir, } static int -pf_check_out(void *arg, struct mbuf **m, struct ifnet *ifp, int dir, +pf_check_out(void *arg, struct mbuf **m, struct ifnet *ifp, int dir, int flags, struct inpcb *inp) { int chk; - chk = pf_test(PF_OUT, ifp, m, inp); + chk = pf_test(PF_OUT, flags, ifp, m, inp); if (chk && *m) { m_freem(*m); *m = NULL; @@ -3697,7 +3697,7 @@ pf_check_out(void *arg, struct mbuf **m, struct ifnet *ifp, int dir, #ifdef INET6 static int -pf_check6_in(void *arg, struct mbuf **m, struct ifnet *ifp, int dir, +pf_check6_in(void *arg, struct mbuf **m, struct ifnet *ifp, int dir, int flags, struct inpcb *inp) { int chk; @@ -3708,7 +3708,7 @@ pf_check6_in(void *arg, struct mbuf **m, struct ifnet *ifp, int dir, * filtering we have change this to lo0 as it is the case in IPv4. */ CURVNET_SET(ifp->if_vnet); - chk = pf_test6(PF_IN, (*m)->m_flags & M_LOOP ? V_loif : ifp, m, inp); + chk = pf_test6(PF_IN, flags, (*m)->m_flags & M_LOOP ? V_loif : ifp, m, inp); CURVNET_RESTORE(); if (chk && *m) { m_freem(*m); @@ -3720,13 +3720,13 @@ pf_check6_in(void *arg, struct mbuf **m, struct ifnet *ifp, int dir, } static int -pf_check6_out(void *arg, struct mbuf **m, struct ifnet *ifp, int dir, +pf_check6_out(void *arg, struct mbuf **m, struct ifnet *ifp, int dir, int flags, struct inpcb *inp) { int chk; CURVNET_SET(ifp->if_vnet); - chk = pf_test6(PF_OUT, ifp, m, inp); + chk = pf_test6(PF_OUT, flags, ifp, m, inp); CURVNET_RESTORE(); if (chk && *m) { m_freem(*m); @@ -3755,22 +3755,22 @@ hook_pf(void) pfh_inet = pfil_head_get(PFIL_TYPE_AF, AF_INET); if (pfh_inet == NULL) return (ESRCH); /* XXX */ - pfil_add_hook(pf_check_in, NULL, PFIL_IN | PFIL_WAITOK, pfh_inet); - pfil_add_hook(pf_check_out, NULL, PFIL_OUT | PFIL_WAITOK, pfh_inet); + pfil_add_hook_flags(pf_check_in, NULL, PFIL_IN | PFIL_WAITOK, pfh_inet); + pfil_add_hook_flags(pf_check_out, NULL, PFIL_OUT | PFIL_WAITOK, pfh_inet); #endif #ifdef INET6 pfh_inet6 = pfil_head_get(PFIL_TYPE_AF, AF_INET6); if (pfh_inet6 == NULL) { #ifdef INET - pfil_remove_hook(pf_check_in, NULL, PFIL_IN | PFIL_WAITOK, + pfil_remove_hook_flags(pf_check_in, NULL, PFIL_IN | PFIL_WAITOK, pfh_inet); - pfil_remove_hook(pf_check_out, NULL, PFIL_OUT | PFIL_WAITOK, + pfil_remove_hook_flags(pf_check_out, NULL, PFIL_OUT | PFIL_WAITOK, pfh_inet); #endif return (ESRCH); /* XXX */ } - pfil_add_hook(pf_check6_in, NULL, PFIL_IN | PFIL_WAITOK, pfh_inet6); - pfil_add_hook(pf_check6_out, NULL, PFIL_OUT | PFIL_WAITOK, pfh_inet6); + pfil_add_hook_flags(pf_check6_in, NULL, PFIL_IN | PFIL_WAITOK, pfh_inet6); + pfil_add_hook_flags(pf_check6_out, NULL, PFIL_OUT | PFIL_WAITOK, pfh_inet6); #endif V_pf_pfil_hooked = 1; @@ -3794,18 +3794,18 @@ dehook_pf(void) pfh_inet = pfil_head_get(PFIL_TYPE_AF, AF_INET); if (pfh_inet == NULL) return (ESRCH); /* XXX */ - pfil_remove_hook(pf_check_in, NULL, PFIL_IN | PFIL_WAITOK, + pfil_remove_hook_flags(pf_check_in, NULL, PFIL_IN | PFIL_WAITOK, pfh_inet); - pfil_remove_hook(pf_check_out, NULL, PFIL_OUT | PFIL_WAITOK, + pfil_remove_hook_flags(pf_check_out, NULL, PFIL_OUT | PFIL_WAITOK, pfh_inet); #endif #ifdef INET6 pfh_inet6 = pfil_head_get(PFIL_TYPE_AF, AF_INET6); if (pfh_inet6 == NULL) return (ESRCH); /* XXX */ - pfil_remove_hook(pf_check6_in, NULL, PFIL_IN | PFIL_WAITOK, + pfil_remove_hook_flags(pf_check6_in, NULL, PFIL_IN | PFIL_WAITOK, pfh_inet6); - pfil_remove_hook(pf_check6_out, NULL, PFIL_OUT | PFIL_WAITOK, + pfil_remove_hook_flags(pf_check6_out, NULL, PFIL_OUT | PFIL_WAITOK, pfh_inet6); #endif diff --git a/freebsd/sys/netpfil/pf/pf_table.c b/freebsd/sys/netpfil/pf/pf_table.c index 68e24d22..06916204 100644 --- a/freebsd/sys/netpfil/pf/pf_table.c +++ b/freebsd/sys/netpfil/pf/pf_table.c @@ -1133,8 +1133,10 @@ pfr_add_tables(struct pfr_table *tbl, int size, int *nadd, int flags) if (p == NULL) senderr(ENOMEM); SLIST_FOREACH(q, &addq, pfrkt_workq) { - if (!pfr_ktable_compare(p, q)) + if (!pfr_ktable_compare(p, q)) { + pfr_destroy_ktable(p, 0); goto _skip; + } } SLIST_INSERT_HEAD(&addq, p, pfrkt_workq); xadd++; diff --git a/freebsd/sys/opencrypto/crypto.c b/freebsd/sys/opencrypto/crypto.c index f6943be2..5db2e872 100644 --- a/freebsd/sys/opencrypto/crypto.c +++ b/freebsd/sys/opencrypto/crypto.c @@ -532,7 +532,7 @@ crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int crid) (*sid) |= (lid & 0xffffffff); cap->cc_sessions++; } else - CRYPTDEB("dev newsession failed"); + CRYPTDEB("dev newsession failed: %d", err); } else { CRYPTDEB("no driver"); err = EOPNOTSUPP; diff --git a/freebsd/sys/opencrypto/cryptodev.c b/freebsd/sys/opencrypto/cryptodev.c index c24af195..3a9ed6e3 100644 --- a/freebsd/sys/opencrypto/cryptodev.c +++ b/freebsd/sys/opencrypto/cryptodev.c @@ -449,6 +449,9 @@ cryptof_ioctl( case CRYPTO_AES_NIST_GCM_16: txform = &enc_xform_aes_nist_gcm; break; + case CRYPTO_CHACHA20: + txform = &enc_xform_chacha20; + break; default: CRYPTDEB("invalid cipher"); @@ -498,6 +501,14 @@ cryptof_ioctl( case CRYPTO_NULL_HMAC: thash = &auth_hash_null; break; + + case CRYPTO_BLAKE2B: + thash = &auth_hash_blake2b; + break; + case CRYPTO_BLAKE2S: + thash = &auth_hash_blake2s; + break; + default: CRYPTDEB("invalid mac"); SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); @@ -873,7 +884,7 @@ cryptodev_op( goto bail; } if ((error = copyin(cop->iv, crde->crd_iv, - cse->txform->blocksize))) { + cse->txform->ivsize))) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } @@ -883,8 +894,8 @@ cryptodev_op( crde->crd_skip = 0; } else if (crde) { crde->crd_flags |= CRD_F_IV_PRESENT; - crde->crd_skip = cse->txform->blocksize; - crde->crd_len -= cse->txform->blocksize; + crde->crd_skip = cse->txform->ivsize; + crde->crd_len -= cse->txform->ivsize; } if (cop->mac && crda == NULL) { @@ -1051,8 +1062,8 @@ cryptodev_aead( crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; } else { crde->crd_flags |= CRD_F_IV_PRESENT; - crde->crd_skip += cse->txform->blocksize; - crde->crd_len -= cse->txform->blocksize; + crde->crd_skip += cse->txform->ivsize; + crde->crd_len -= cse->txform->ivsize; } if ((error = copyin(caead->tag, (caddr_t)cod->uio.uio_iov[0].iov_base + diff --git a/freebsd/sys/opencrypto/cryptodev.h b/freebsd/sys/opencrypto/cryptodev.h index f929d10a..65422541 100644 --- a/freebsd/sys/opencrypto/cryptodev.h +++ b/freebsd/sys/opencrypto/cryptodev.h @@ -112,7 +112,8 @@ #define AES_ICM_BLOCK_LEN 1 #define ARC4_BLOCK_LEN 1 #define CAMELLIA_BLOCK_LEN 16 -#define EALG_MAX_BLOCK_LEN AES_BLOCK_LEN /* Keep this updated */ +#define CHACHA20_NATIVE_BLOCK_LEN 64 +#define EALG_MAX_BLOCK_LEN CHACHA20_NATIVE_BLOCK_LEN /* Keep this updated */ /* IV Lengths */ @@ -178,7 +179,10 @@ #define CRYPTO_AES_128_NIST_GMAC 26 /* auth side */ #define CRYPTO_AES_192_NIST_GMAC 27 /* auth side */ #define CRYPTO_AES_256_NIST_GMAC 28 /* auth side */ -#define CRYPTO_ALGORITHM_MAX 28 /* Keep updated - see below */ +#define CRYPTO_BLAKE2B 29 /* Blake2b hash */ +#define CRYPTO_BLAKE2S 30 /* Blake2s hash */ +#define CRYPTO_CHACHA20 31 /* Chacha20 stream cipher */ +#define CRYPTO_ALGORITHM_MAX 31 /* Keep updated - see below */ #define CRYPTO_ALGO_VALID(x) ((x) >= CRYPTO_ALGORITHM_MIN && \ (x) <= CRYPTO_ALGORITHM_MAX) @@ -346,10 +350,11 @@ struct cryptostats { #ifdef _KERNEL #if 0 -#define CRYPTDEB(s) do { printf("%s:%d: %s\n", __FILE__, __LINE__, s); \ - } while (0) +#define CRYPTDEB(s, ...) do { \ + printf("%s:%d: " s "\n", __FILE__, __LINE__, ## __VA_ARGS__); \ +} while (0) #else -#define CRYPTDEB(s) do { } while (0) +#define CRYPTDEB(...) do { } while (0) #endif /* Standard initialization structure beginning */ diff --git a/freebsd/sys/opencrypto/cryptosoft.c b/freebsd/sys/opencrypto/cryptosoft.c index b2828452..55e98b13 100644 --- a/freebsd/sys/opencrypto/cryptosoft.c +++ b/freebsd/sys/opencrypto/cryptosoft.c @@ -250,21 +250,29 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, break; } - /* - * Warning: idat may point to garbage here, but - * we only use it in the while() loop, only if - * there are indeed enough data. - */ - idat = (char *)uio->uio_iov[ind].iov_base + k; - while (uio->uio_iov[ind].iov_len >= k + blks && i > 0) { + size_t nb, rem; + + nb = blks; + rem = uio->uio_iov[ind].iov_len - k; + idat = (char *)uio->uio_iov[ind].iov_base + k; + if (exf->reinit) { - if (crd->crd_flags & CRD_F_ENCRYPT) { + if ((crd->crd_flags & CRD_F_ENCRYPT) != 0 && + exf->encrypt_multi == NULL) exf->encrypt(sw->sw_kschedule, idat); - } else { + else if ((crd->crd_flags & CRD_F_ENCRYPT) != 0) { + nb = rounddown(rem, blks); + exf->encrypt_multi(sw->sw_kschedule, + idat, nb); + } else if (exf->decrypt_multi == NULL) exf->decrypt(sw->sw_kschedule, idat); + else { + nb = rounddown(rem, blks); + exf->decrypt_multi(sw->sw_kschedule, + idat, nb); } } else if (crd->crd_flags & CRD_F_ENCRYPT) { /* XOR with previous block/IV */ @@ -290,10 +298,10 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, ivp = nivp; } - idat += blks; - count += blks; - k += blks; - i -= blks; + idat += nb; + count += nb; + k += nb; + i -= nb; } /* @@ -374,6 +382,11 @@ swcr_authprepare(struct auth_hash *axf, struct swcr_data *sw, u_char *key, axf->Final(buf, sw->sw_ictx); break; } + case CRYPTO_BLAKE2B: + case CRYPTO_BLAKE2S: + axf->Setkey(sw->sw_ictx, key, klen); + axf->Init(sw->sw_ictx); + break; default: printf("%s: CRD_F_KEY_EXPLICIT flag given, but algorithm %d " "doesn't use keys.\n", __func__, axf->type); @@ -440,6 +453,8 @@ swcr_authcompute(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, axf->Final(aalg, &ctx); break; + case CRYPTO_BLAKE2B: + case CRYPTO_BLAKE2S: case CRYPTO_NULL_HMAC: axf->Final(aalg, &ctx); break; @@ -561,14 +576,26 @@ swcr_authenc(struct cryptop *crp) exf->reinit(swe->sw_kschedule, iv); /* Do encryption/decryption with MAC */ - for (i = 0; i < crde->crd_len; i += blksz) { - len = MIN(crde->crd_len - i, blksz); + for (i = 0; i < crde->crd_len; i += len) { + if (exf->encrypt_multi != NULL) { + len = rounddown(crde->crd_len - i, blksz); + if (len == 0) + len = blksz; + else + len = MIN(len, sizeof(blkbuf)); + } else + len = blksz; + len = MIN(crde->crd_len - i, len); if (len < blksz) bzero(blk, blksz); crypto_copydata(crp->crp_flags, buf, crde->crd_skip + i, len, blk); if (crde->crd_flags & CRD_F_ENCRYPT) { - exf->encrypt(swe->sw_kschedule, blk); + if (exf->encrypt_multi != NULL) + exf->encrypt_multi(swe->sw_kschedule, blk, + len); + else + exf->encrypt(swe->sw_kschedule, blk); axf->Update(&ctx, blk, len); crypto_copyback(crp->crp_flags, buf, crde->crd_skip + i, len, blk); @@ -805,6 +832,9 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri) case CRYPTO_NULL_CBC: txf = &enc_xform_null; goto enccommon; + case CRYPTO_CHACHA20: + txf = &enc_xform_chacha20; + goto enccommon; enccommon: if (cri->cri_key != NULL) { error = txf->setkey(&((*swd)->sw_kschedule), @@ -948,6 +978,25 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri) (*swd)->sw_axf = axf; break; + case CRYPTO_BLAKE2B: + axf = &auth_hash_blake2b; + goto auth5common; + case CRYPTO_BLAKE2S: + axf = &auth_hash_blake2s; + auth5common: + (*swd)->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, + M_NOWAIT); + if ((*swd)->sw_ictx == NULL) { + swcr_freesession_locked(dev, i); + rw_runlock(&swcr_sessions_lock); + return ENOBUFS; + } + axf->Setkey((*swd)->sw_ictx, cri->cri_key, + cri->cri_klen / 8); + axf->Init((*swd)->sw_ictx); + (*swd)->sw_axf = axf; + break; + case CRYPTO_DEFLATE_COMP: cxf = &comp_algo_deflate; (*swd)->sw_cxf = cxf; @@ -1012,6 +1061,7 @@ swcr_freesession_locked(device_t dev, u_int64_t tid) case CRYPTO_AES_NIST_GMAC: case CRYPTO_CAMELLIA_CBC: case CRYPTO_NULL_CBC: + case CRYPTO_CHACHA20: txf = swd->sw_exf; if (swd->sw_kschedule) @@ -1051,12 +1101,16 @@ swcr_freesession_locked(device_t dev, u_int64_t tid) } break; + case CRYPTO_BLAKE2B: + case CRYPTO_BLAKE2S: case CRYPTO_MD5: case CRYPTO_SHA1: axf = swd->sw_axf; - if (swd->sw_ictx) + if (swd->sw_ictx) { + explicit_bzero(swd->sw_ictx, axf->ctxsize); free(swd->sw_ictx, M_CRYPTO_DATA); + } break; case CRYPTO_DEFLATE_COMP: @@ -1137,6 +1191,7 @@ swcr_process(device_t dev, struct cryptop *crp, int hint) case CRYPTO_AES_XTS: case CRYPTO_AES_ICM: case CRYPTO_CAMELLIA_CBC: + case CRYPTO_CHACHA20: if ((crp->crp_etype = swcr_encdec(crd, sw, crp->crp_buf, crp->crp_flags)) != 0) goto done; @@ -1155,6 +1210,8 @@ swcr_process(device_t dev, struct cryptop *crp, int hint) case CRYPTO_SHA1_KPDK: case CRYPTO_MD5: case CRYPTO_SHA1: + case CRYPTO_BLAKE2B: + case CRYPTO_BLAKE2S: if ((crp->crp_etype = swcr_authcompute(crd, sw, crp->crp_buf, crp->crp_flags)) != 0) goto done; @@ -1246,6 +1303,9 @@ swcr_attach(device_t dev) REGISTER(CRYPTO_AES_256_NIST_GMAC); REGISTER(CRYPTO_CAMELLIA_CBC); REGISTER(CRYPTO_DEFLATE_COMP); + REGISTER(CRYPTO_BLAKE2B); + REGISTER(CRYPTO_BLAKE2S); + REGISTER(CRYPTO_CHACHA20); #undef REGISTER return 0; diff --git a/freebsd/sys/opencrypto/xform_auth.h b/freebsd/sys/opencrypto/xform_auth.h index c51f9400..74c6d063 100644 --- a/freebsd/sys/opencrypto/xform_auth.h +++ b/freebsd/sys/opencrypto/xform_auth.h @@ -75,6 +75,8 @@ extern struct auth_hash auth_hash_hmac_sha2_512; extern struct auth_hash auth_hash_nist_gmac_aes_128; extern struct auth_hash auth_hash_nist_gmac_aes_192; extern struct auth_hash auth_hash_nist_gmac_aes_256; +extern struct auth_hash auth_hash_blake2b; +extern struct auth_hash auth_hash_blake2s; union authctx { MD5_CTX md5ctx; diff --git a/freebsd/sys/opencrypto/xform_enc.h b/freebsd/sys/opencrypto/xform_enc.h index 7bb6a9dd..545e0ec2 100644 --- a/freebsd/sys/opencrypto/xform_enc.h +++ b/freebsd/sys/opencrypto/xform_enc.h @@ -51,7 +51,7 @@ struct enc_xform { int type; char *name; - u_int16_t blocksize; + u_int16_t blocksize; /* Required input block size -- 1 for stream ciphers. */ u_int16_t ivsize; u_int16_t minkey, maxkey; void (*encrypt) (caddr_t, u_int8_t *); @@ -59,6 +59,14 @@ struct enc_xform { int (*setkey) (u_int8_t **, u_int8_t *, int len); void (*zerokey) (u_int8_t **); void (*reinit) (caddr_t, u_int8_t *); + /* + * Encrypt/decrypt 1+ blocks of input -- total size is 'len' bytes. + * Len is guaranteed to be a multiple of the defined 'blocksize'. + * Optional interface -- most useful for stream ciphers with a small + * blocksize (1). + */ + void (*encrypt_multi) (void *, uint8_t *, size_t len); + void (*decrypt_multi) (void *, uint8_t *, size_t len); }; @@ -75,6 +83,7 @@ extern struct enc_xform enc_xform_aes_nist_gmac; extern struct enc_xform enc_xform_aes_xts; extern struct enc_xform enc_xform_arc4; extern struct enc_xform enc_xform_camellia; +extern struct enc_xform enc_xform_chacha20; struct aes_icm_ctx { u_int32_t ac_ek[4*(RIJNDAEL_MAXNR + 1)]; diff --git a/freebsd/sys/powerpc/include/machine/psl.h b/freebsd/sys/powerpc/include/machine/psl.h index 329120f4..1179d744 100644 --- a/freebsd/sys/powerpc/include/machine/psl.h +++ b/freebsd/sys/powerpc/include/machine/psl.h @@ -90,28 +90,13 @@ #define PSL_FE_PREC (PSL_FE0 | PSL_FE1) /* precise */ #define PSL_FE_DFLT PSL_FE_DIS /* default == none */ -#if defined(BOOKE_E500) -/* Initial kernel MSR, use IS=1 ad DS=1. */ -#define PSL_KERNSET_INIT (PSL_IS | PSL_DS) +#ifndef LOCORE +extern register_t psl_kernset; /* Default MSR values for kernel */ +extern register_t psl_userset; /* Default MSR values for userland */ #ifdef __powerpc64__ -#define PSL_KERNSET (PSL_CM | PSL_CE | PSL_ME | PSL_EE) -#else -#define PSL_KERNSET (PSL_CE | PSL_ME | PSL_EE) +extern register_t psl_userset32; /* Default user MSR values for 32-bit */ #endif -#define PSL_SRR1_MASK 0x00000000UL /* No mask on Book-E */ -#elif defined(BOOKE_PPC4XX) -#define PSL_KERNSET (PSL_CE | PSL_ME | PSL_EE | PSL_FP) -#define PSL_SRR1_MASK 0x00000000UL /* No mask on Book-E */ -#elif defined(AIM) -#ifdef __powerpc64__ -#define PSL_KERNSET (PSL_SF | PSL_EE | PSL_ME | PSL_IR | PSL_DR | PSL_RI) -#else -#define PSL_KERNSET (PSL_EE | PSL_ME | PSL_IR | PSL_DR | PSL_RI) -#endif -#define PSL_SRR1_MASK 0x783f0000UL /* Bits 1-4, 10-15 (ppc32), 33-36, 42-47 (ppc64) */ +extern register_t psl_userstatic; /* Bits of SRR1 userland may not set */ #endif -#define PSL_USERSET (PSL_KERNSET | PSL_PR) -#define PSL_USERSTATIC (~(PSL_VEC | PSL_FP | PSL_FE0 | PSL_FE1) & ~PSL_SRR1_MASK) - #endif /* _MACHINE_PSL_H_ */ diff --git a/freebsd/sys/powerpc/include/machine/spr.h b/freebsd/sys/powerpc/include/machine/spr.h index db94753b..24ae5fe9 100644 --- a/freebsd/sys/powerpc/include/machine/spr.h +++ b/freebsd/sys/powerpc/include/machine/spr.h @@ -170,6 +170,7 @@ #define IBMPOWER3PLUS 0x0041 #define IBM970MP 0x0044 #define IBM970GX 0x0045 +#define IBMPOWERPCA2 0x0049 #define IBMPOWER7PLUS 0x004a #define IBMPOWER8E 0x004b #define IBMPOWER8 0x004d @@ -201,7 +202,14 @@ #define SPR_LPCR 0x13e /* Logical Partitioning Control */ #define LPCR_LPES 0x008 /* Bit 60 */ - +#define LPCR_PECE_DRBL (1ULL << 16) /* Directed Privileged Doorbell */ +#define LPCR_PECE_HDRBL (1ULL << 15) /* Directed Hypervisor Doorbell */ +#define LPCR_PECE_EXT (1ULL << 14) /* External exceptions */ +#define LPCR_PECE_DECR (1ULL << 13) /* Decrementer exceptions */ +#define LPCR_PECE_ME (1ULL << 12) /* Machine Check and Hypervisor */ + /* Maintenance exceptions */ +#define LPCR_PECE_WAKESET (LPCR_PECE_EXT | LPCR_PECE_DECR | LPCR_PECE_ME) + #define SPR_EPCR 0x133 #define EPCR_EXTGS 0x80000000 #define EPCR_DTLBGS 0x40000000 diff --git a/freebsd/sys/sys/aio.h b/freebsd/sys/sys/aio.h index 2843042b..4963e986 100644 --- a/freebsd/sys/sys/aio.h +++ b/freebsd/sys/sys/aio.h @@ -55,7 +55,7 @@ #ifndef __rtems__ /* - * Maximum number of allowed LIO operations + * Maximum number of operations in a single lio_listio call */ #define AIO_LISTIO_MAX 16 #endif /* __rtems__ */ diff --git a/freebsd/sys/sys/buf.h b/freebsd/sys/sys/buf.h index 77acac5b..c9b2eeae 100644 --- a/freebsd/sys/sys/buf.h +++ b/freebsd/sys/sys/buf.h @@ -112,7 +112,9 @@ struct buf { off_t b_offset; /* Offset into file. */ TAILQ_ENTRY(buf) b_bobufs; /* (V) Buffer's associated vnode. */ uint32_t b_vflags; /* (V) BV_* flags */ - unsigned short b_qindex; /* (Q) buffer queue index */ + uint8_t b_qindex; /* (Q) buffer queue index */ + uint8_t b_domain; /* (Q) buf domain this resides in */ + uint16_t b_subqueue; /* (Q) per-cpu q if any */ uint32_t b_flags; /* B_* flags. */ b_xflags_t b_xflags; /* extra flags */ struct lock b_lock; /* Buffer lock */ @@ -217,7 +219,7 @@ struct buf { #define B_DONE 0x00000200 /* I/O completed. */ #define B_EINTR 0x00000400 /* I/O was interrupted */ #define B_NOREUSE 0x00000800 /* Contents not reused once released. */ -#define B_00001000 0x00001000 /* Available flag. */ +#define B_REUSE 0x00001000 /* Contents reused, second chance. */ #define B_INVAL 0x00002000 /* Does not contain valid info. */ #define B_BARRIER 0x00004000 /* Write this and all preceding first. */ #define B_NOCACHE 0x00008000 /* Do not cache block after use. */ @@ -241,7 +243,7 @@ struct buf { #define PRINT_BUF_FLAGS "\20\40remfree\37cluster\36vmio\35ram\34managed" \ "\33paging\32infreecnt\31nocopy\30b23\27relbuf\26b21\25b20" \ "\24b19\23b18\22clusterok\21malloc\20nocache\17b14\16inval" \ - "\15b12\14noreuse\13eintr\12done\11b8\10delwri" \ + "\15reuse\14noreuse\13eintr\12done\11b8\10delwri" \ "\7validsuspwrt\6cache\5deferred\4direct\3async\2needcommit\1age" /* @@ -542,7 +544,6 @@ struct buf *geteblk(int, int); int bufwait(struct buf *); int bufwrite(struct buf *); void bufdone(struct buf *); -void bufdone_finish(struct buf *); void bd_speedup(void); int cluster_read(struct vnode *, u_quad_t, daddr_t, long, diff --git a/freebsd/sys/sys/bufobj.h b/freebsd/sys/sys/bufobj.h index fb3f0873..b02d4276 100644 --- a/freebsd/sys/sys/bufobj.h +++ b/freebsd/sys/sys/bufobj.h @@ -106,6 +106,7 @@ struct bufobj { struct bufv bo_dirty; /* i Dirty buffers */ long bo_numoutput; /* i Writes in progress */ u_int bo_flag; /* i Flags */ + int bo_domain; /* - Clean queue affinity */ int bo_bsize; /* - Block size for i/o */ }; @@ -126,6 +127,7 @@ struct bufobj { #define ASSERT_BO_LOCKED(bo) rw_assert(BO_LOCKPTR((bo)), RA_LOCKED) #define ASSERT_BO_UNLOCKED(bo) rw_assert(BO_LOCKPTR((bo)), RA_UNLOCKED) +void bufobj_init(struct bufobj *bo, void *private); void bufobj_wdrop(struct bufobj *bo); void bufobj_wref(struct bufobj *bo); void bufobj_wrefl(struct bufobj *bo); diff --git a/freebsd/sys/sys/bus.h b/freebsd/sys/sys/bus.h index 7a3f6b1a..d1d6bbb9 100644 --- a/freebsd/sys/sys/bus.h +++ b/freebsd/sys/sys/bus.h @@ -702,6 +702,7 @@ void bus_data_generation_update(void); #define BUS_PASS_INTERRUPT 40 /* Interrupt controllers. */ #define BUS_PASS_TIMER 50 /* Timers and clocks. */ #define BUS_PASS_SCHEDULER 60 /* Start scheduler. */ +#define BUS_PASS_SUPPORTDEV 100000 /* Drivers which support DEFAULT drivers. */ #define BUS_PASS_DEFAULT __INT_MAX /* Everything else. */ #define BUS_PASS_ORDER_FIRST 0 diff --git a/freebsd/sys/sys/bus_dma.h b/freebsd/sys/sys/bus_dma.h index 2bf46ca8..eb2bc42d 100644 --- a/freebsd/sys/sys/bus_dma.h +++ b/freebsd/sys/sys/bus_dma.h @@ -178,7 +178,7 @@ int bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, /* * Set the memory domain to be used for allocations. - * + * * Automatic for PCI devices. Must be set prior to creating maps or * allocating memory. */ diff --git a/freebsd/sys/sys/callout.h b/freebsd/sys/sys/callout.h index a0b45d98..e5e5df85 100644 --- a/freebsd/sys/sys/callout.h +++ b/freebsd/sys/sys/callout.h @@ -62,6 +62,7 @@ #define C_HARDCLOCK 0x0100 /* align to hardclock() calls */ #define C_ABSOLUTE 0x0200 /* event time is absolute. */ #define C_PRECALC 0x0400 /* event time is pre-calculated. */ +#define C_CATCH 0x0800 /* catch signals, used by pause_sbt(9) */ struct callout_handle { struct callout *callout; diff --git a/freebsd/sys/sys/disk.h b/freebsd/sys/sys/disk.h new file mode 100644 index 00000000..020626e2 --- /dev/null +++ b/freebsd/sys/sys/disk.h @@ -0,0 +1,161 @@ +/*- + * SPDX-License-Identifier: Beerware + * + * ---------------------------------------------------------------------------- + * "THE BEER-WARE LICENSE" (Revision 42): + * <phk@FreeBSD.ORG> wrote this file. As long as you retain this notice you + * can do whatever you want with this stuff. If we meet some day, and you think + * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp + * ---------------------------------------------------------------------------- + * + * $FreeBSD$ + * + */ + +#ifndef _SYS_DISK_H_ +#define _SYS_DISK_H_ + +#include <sys/ioccom.h> +#include <sys/kerneldump.h> +#include <sys/types.h> +#include <sys/disk_zone.h> + +#ifdef _KERNEL + +#ifndef _SYS_CONF_H_ +#include <sys/conf.h> /* XXX: temporary to avoid breakage */ +#endif + +void disk_err(struct bio *bp, const char *what, int blkdone, int nl); + +#endif + +#define DIOCGSECTORSIZE _IOR('d', 128, u_int) + /* + * Get the sector size of the device in bytes. The sector size is the + * smallest unit of data which can be transferred from this device. + * Usually this is a power of 2 but it might not be (i.e. CDROM audio). + */ + +#define DIOCGMEDIASIZE _IOR('d', 129, off_t) /* Get media size in bytes */ + /* + * Get the size of the entire device in bytes. This should be a + * multiple of the sector size. + */ + +#define DIOCGFWSECTORS _IOR('d', 130, u_int) /* Get firmware's sectorcount */ + /* + * Get the firmware's notion of number of sectors per track. This + * value is mostly used for compatibility with various ill designed + * disk label formats. Don't use it unless you have to. + */ + +#define DIOCGFWHEADS _IOR('d', 131, u_int) /* Get firmware's headcount */ + /* + * Get the firmwares notion of number of heads per cylinder. This + * value is mostly used for compatibility with various ill designed + * disk label formats. Don't use it unless you have to. + */ + +#define DIOCSKERNELDUMP_FREEBSD11 _IOW('d', 133, u_int) + /* + * Enable/Disable (the argument is boolean) the device for kernel + * core dumps. + */ + +#define DIOCGFRONTSTUFF _IOR('d', 134, off_t) + /* + * Many disk formats have some amount of space reserved at the + * start of the disk to hold bootblocks, various disklabels and + * similar stuff. This ioctl returns the number of such bytes + * which may apply to the device. + */ + +#define DIOCGFLUSH _IO('d', 135) /* Flush write cache */ + /* + * Flush write cache of the device. + */ + +#define DIOCGDELETE _IOW('d', 136, off_t[2]) /* Delete data */ + /* + * Mark data on the device as unused. + */ + +#define DISK_IDENT_SIZE 256 +#define DIOCGIDENT _IOR('d', 137, char[DISK_IDENT_SIZE]) + /*- + * Get the ident of the given provider. Ident is (most of the time) + * a uniqe and fixed provider's identifier. Ident's properties are as + * follow: + * - ident value is preserved between reboots, + * - provider can be detached/attached and ident is preserved, + * - provider's name can change - ident can't, + * - ident value should not be based on on-disk metadata; in other + * words copying whole data from one disk to another should not + * yield the same ident for the other disk, + * - there could be more than one provider with the same ident, but + * only if they point at exactly the same physical storage, this is + * the case for multipathing for example, + * - GEOM classes that consumes single providers and provide single + * providers, like geli, gbde, should just attach class name to the + * ident of the underlying provider, + * - ident is an ASCII string (is printable), + * - ident is optional and applications can't relay on its presence. + */ + +#define DIOCGPROVIDERNAME _IOR('d', 138, char[MAXPATHLEN]) + /* + * Store the provider name, given a device path, in a buffer. The buffer + * must be at least MAXPATHLEN bytes long. + */ + +#define DIOCGSTRIPESIZE _IOR('d', 139, off_t) /* Get stripe size in bytes */ + /* + * Get the size of the device's optimal access block in bytes. + * This should be a multiple of the sector size. + */ + +#define DIOCGSTRIPEOFFSET _IOR('d', 140, off_t) /* Get stripe offset in bytes */ + /* + * Get the offset of the first device's optimal access block in bytes. + * This should be a multiple of the sector size. + */ + +#define DIOCGPHYSPATH _IOR('d', 141, char[MAXPATHLEN]) + /* + * Get a string defining the physical path for a given provider. + * This has similar rules to ident, but is intended to uniquely + * identify the physical location of the device, not the current + * occupant of that location. + */ + +struct diocgattr_arg { + char name[64]; + int len; + union { + char str[DISK_IDENT_SIZE]; + off_t off; + int i; + uint16_t u16; + } value; +}; +#define DIOCGATTR _IOWR('d', 142, struct diocgattr_arg) + +#define DIOCZONECMD _IOWR('d', 143, struct disk_zone_args) + +#ifndef __rtems__ +struct diocskerneldump_arg { + uint8_t kda_enable; + uint8_t kda_compression; + uint8_t kda_encryption; + uint8_t kda_key[KERNELDUMP_KEY_MAX_SIZE]; + uint32_t kda_encryptedkeysize; + uint8_t *kda_encryptedkey; +}; +#define DIOCSKERNELDUMP _IOW('d', 144, struct diocskerneldump_arg) + /* + * Enable/Disable the device for kernel core dumps. + */ +#endif /* __rtems__ */ + +#endif /* _SYS_DISK_H_ */ diff --git a/freebsd/sys/sys/domainset.h b/freebsd/sys/sys/domainset.h index 6580e1ed..81375ed0 100644 --- a/freebsd/sys/sys/domainset.h +++ b/freebsd/sys/sys/domainset.h @@ -28,8 +28,8 @@ * $FreeBSD$ */ -#ifndef _SYS_DOMAINSETSET_H_ -#define _SYS_DOMAINSETSET_H_ +#ifndef _SYS_DOMAINSET_H_ +#define _SYS_DOMAINSET_H_ #include <sys/_domainset.h> @@ -38,7 +38,11 @@ #define _NDOMAINSETBITS _BITSET_BITS #define _NDOMAINSETWORDS __bitset_words(DOMAINSET_SETSIZE) -#define DOMAINSETSETBUFSIZ ((2 + sizeof(long) * 2) * _NDOMAINSETWORDS) +#define DOMAINSETBUFSIZ \ + (((2 + sizeof(long) * 2) * _NDOMAINSETWORDS) + \ + sizeof("::") + sizeof(__XSTRING(DOMAINSET_POLICY_MAX)) + \ + sizeof(__XSTRING(MAXMEMDOM))) + #define DOMAINSET_CLR(n, p) BIT_CLR(DOMAINSET_SETSIZE, n, p) #define DOMAINSET_COPY(f, t) BIT_COPY(DOMAINSET_SETSIZE, f, t) @@ -73,23 +77,37 @@ #define DOMAINSET_POLICY_ROUNDROBIN 1 #define DOMAINSET_POLICY_FIRSTTOUCH 2 #define DOMAINSET_POLICY_PREFER 3 -#define DOMAINSET_POLICY_MAX DOMAINSET_POLICY_PREFER +#define DOMAINSET_POLICY_INTERLEAVE 4 +#define DOMAINSET_POLICY_MAX DOMAINSET_POLICY_INTERLEAVE #ifdef _KERNEL -#include <sys/queue.h> -LIST_HEAD(domainlist, domainset); +#if MAXMEMDOM < 256 +typedef uint8_t domainid_t; +#else +typedef uint16_t domainid_t; +#endif struct domainset { LIST_ENTRY(domainset) ds_link; domainset_t ds_mask; /* allowed domains. */ uint16_t ds_policy; /* Policy type. */ - int16_t ds_prefer; /* Preferred domain or -1. */ - uint16_t ds_cnt; /* popcnt from above. */ - uint16_t ds_max; /* Maximum domain in set. */ + domainid_t ds_prefer; /* Preferred domain or -1. */ + domainid_t ds_cnt; /* popcnt from above. */ + domainid_t ds_order[MAXMEMDOM]; /* nth domain table. */ }; void domainset_zero(void); +/* + * Add a domainset to the system based on a key initializing policy, prefer, + * and mask. Do not create and directly use domainset structures. The + * returned value will not match the key pointer. + */ +struct domainset *domainset_create(const struct domainset *); +#ifdef _SYS_SYSCTL_H_ +int sysctl_handle_domainset(SYSCTL_HANDLER_ARGS); +#endif + #else __BEGIN_DECLS int cpuset_getdomain(cpulevel_t, cpuwhich_t, id_t, size_t, domainset_t *, @@ -99,4 +117,4 @@ int cpuset_setdomain(cpulevel_t, cpuwhich_t, id_t, size_t, __END_DECLS #endif -#endif /* !_SYS_DOMAINSETSET_H_ */ +#endif /* !_SYS_DOMAINSET_H_ */ diff --git a/freebsd/sys/sys/kernel.h b/freebsd/sys/sys/kernel.h index 81ceca04..42a37604 100644 --- a/freebsd/sys/sys/kernel.h +++ b/freebsd/sys/sys/kernel.h @@ -270,7 +270,7 @@ sysinit_tslog_shim(const void * data) sysinit_tslog_shim, \ &uniquifier ## _sys_init_tslog \ }; \ - DATA_SET(sysinit_set,uniquifier ## _sys_init) + DATA_WSET(sysinit_set,uniquifier ## _sys_init) #else #ifndef __rtems__ #define C_SYSINIT(uniquifier, subsystem, order, func, ident) \ @@ -280,7 +280,7 @@ sysinit_tslog_shim(const void * data) func, \ (ident) \ }; \ - DATA_SET(sysinit_set,uniquifier ## _sys_init) + DATA_WSET(sysinit_set,uniquifier ## _sys_init) #else /* __rtems__ */ #define SYSINIT_ENTRY_NAME(uniquifier) \ _bsd_ ## uniquifier ## _sys_init @@ -293,7 +293,7 @@ sysinit_tslog_shim(const void * data) func, \ (ident) \ }; \ - RWDATA_SET(sysinit_set,SYSINIT_ENTRY_NAME(uniquifier)) + DATA_WSET(sysinit_set,SYSINIT_ENTRY_NAME(uniquifier)) #define SYSINIT_REFERENCE(uniquifier) \ extern struct sysinit SYSINIT_ENTRY_NAME(uniquifier); \ static struct sysinit const * const \ @@ -323,7 +323,7 @@ sysinit_tslog_shim(const void * data) func, \ (ident) \ }; \ - DATA_SET(sysuninit_set,uniquifier ## _sys_uninit) + DATA_WSET(sysuninit_set,uniquifier ## _sys_uninit) #else /* __rtems__ */ #define C_SYSUNINIT(uniquifier, subsystem, order, func, ident) #endif /* __rtems__ */ diff --git a/freebsd/sys/sys/ktr_class.h b/freebsd/sys/sys/ktr_class.h index 9190ce27..32d905d2 100644 --- a/freebsd/sys/sys/ktr_class.h +++ b/freebsd/sys/sys/ktr_class.h @@ -65,7 +65,7 @@ #define KTR_VM 0x00100000 /* The virtual memory system */ #define KTR_INET 0x00200000 /* IPv4 stack */ #define KTR_RUNQ 0x00400000 /* Run queue */ -#define KTR_CONTENTION 0x00800000 /* Lock contention */ +#define KTR_SPARE5 0x00800000 #define KTR_UMA 0x01000000 /* UMA slab allocator */ #define KTR_CALLOUT 0x02000000 /* Callouts and timeouts */ #define KTR_GEOM 0x04000000 /* GEOM I/O events */ diff --git a/freebsd/sys/sys/linker_set.h b/freebsd/sys/sys/linker_set.h index 7e33c93f..e10be24c 100755 --- a/freebsd/sys/sys/linker_set.h +++ b/freebsd/sys/sys/linker_set.h @@ -60,12 +60,13 @@ */ #ifdef __GNUCLIKE___SECTION #ifndef __rtems__ -#define __MAKE_SET(set, sym) \ +#define __MAKE_SET_QV(set, sym, qv) \ __GLOBL(__CONCAT(__start_set_,set)); \ __GLOBL(__CONCAT(__stop_set_,set)); \ - static void const * __MAKE_SET_CONST \ + static void const * qv \ __set_##set##_sym_##sym __section("set_" #set) \ __used = &(sym) +#define __MAKE_SET(set, sym) __MAKE_SET_QV(set, sym, __MAKE_SET_CONST) #else /* __rtems__ */ #define RTEMS_BSD_DEFINE_SET(set, type) \ type const __CONCAT(_bsd__start_set_,set)[0] \ @@ -114,12 +115,14 @@ */ #define TEXT_SET(set, sym) __MAKE_SET(set, sym) #define DATA_SET(set, sym) __MAKE_SET(set, sym) +#ifndef __rtems__ +#define DATA_WSET(set, sym) __MAKE_SET_QV(set, sym, ) +#else /* __rtems__ */ +#define DATA_WSET(set, sym) __MAKE_RWSET(set, sym) +#endif /* __rtems__ */ #define BSS_SET(set, sym) __MAKE_SET(set, sym) #define ABS_SET(set, sym) __MAKE_SET(set, sym) #define SET_ENTRY(set, sym) __MAKE_SET(set, sym) -#ifdef __rtems__ -#define RWDATA_SET(set, sym) __MAKE_RWSET(set, sym) -#endif /* __rtems__ */ /* * Initialize before referring to a given linker set. diff --git a/freebsd/sys/sys/lockstat.h b/freebsd/sys/sys/lockstat.h index 50747e89..6167a7d1 100644 --- a/freebsd/sys/sys/lockstat.h +++ b/freebsd/sys/sys/lockstat.h @@ -109,12 +109,7 @@ extern volatile int lockstat_enabled; LOCKSTAT_RECORD1(probe, lp, a); \ } while (0) -#ifndef LOCK_PROFILING #define LOCKSTAT_PROFILE_ENABLED(probe) __predict_false(lockstat_enabled) -#define LOCKSTAT_OOL_PROFILE_ENABLED(probe) LOCKSTAT_PROFILE_ENABLED(probe) -#else -#define LOCKSTAT_OOL_PROFILE_ENABLED(probe) 1 -#endif struct lock_object; uint64_t lockstat_nsecs(struct lock_object *); @@ -139,10 +134,7 @@ uint64_t lockstat_nsecs(struct lock_object *); #define LOCKSTAT_PROFILE_RELEASE_RWLOCK(probe, lp, a) \ LOCKSTAT_PROFILE_RELEASE_LOCK(probe, lp) -#ifndef LOCK_PROFILING #define LOCKSTAT_PROFILE_ENABLED(probe) 0 -#endif -#define LOCKSTAT_OOL_PROFILE_ENABLED(probe) 1 #endif /* !KDTRACE_HOOKS */ diff --git a/freebsd/sys/sys/module.h b/freebsd/sys/sys/module.h index eb8b03d8..c3f8dc3c 100644 --- a/freebsd/sys/sys/module.h +++ b/freebsd/sys/sys/module.h @@ -201,7 +201,7 @@ struct mod_pnp_match_info * D pointer to a string to human readable description for device * P A pointer that should be ignored * E EISA PNP Identifier (in binary, but bus publishes string) - * K Key for whole table. pnp_name=value. must be last, if present. + * T Key for whole table. pnp_name=value. must be last, if present. * * The pnp_name "#" is reserved for other fields that should be ignored. * Otherwise pnp_name must match the name from the parent device's pnpinfo diff --git a/freebsd/sys/sys/mutex.h b/freebsd/sys/sys/mutex.h index 52587064..e15de1ae 100644 --- a/freebsd/sys/sys/mutex.h +++ b/freebsd/sys/sys/mutex.h @@ -133,6 +133,8 @@ int __mtx_trylock_spin_flags(volatile uintptr_t *c, int opts, const char *file, int line); void __mtx_unlock_spin_flags(volatile uintptr_t *c, int opts, const char *file, int line); +void mtx_spin_wait_unlocked(struct mtx *m); + #else /* __rtems__ */ void mtx_init(struct mtx *m, const char *name, const char *type, int opts); void mtx_destroy(struct mtx *m); diff --git a/freebsd/sys/sys/priv.h b/freebsd/sys/sys/priv.h index 52f1dbd2..55453f5a 100644 --- a/freebsd/sys/sys/priv.h +++ b/freebsd/sys/sys/priv.h @@ -344,7 +344,8 @@ #define PRIV_NET_SETIFDESCR 418 /* Set interface description. */ #define PRIV_NET_SETIFFIB 419 /* Set interface fib. */ #define PRIV_NET_VXLAN 420 /* Administer vxlan. */ -#define PRIV_NET_SETVLANPCP 421 /* Set VLAN priority. */ +#define PRIV_NET_SETLANPCP 421 /* Set LAN priority. */ +#define PRIV_NET_SETVLANPCP PRIV_NET_SETLANPCP /* Alias Set VLAN priority */ /* * 802.11-related privileges. diff --git a/freebsd/sys/sys/proc.h b/freebsd/sys/sys/proc.h index ab292769..cef3ae2e 100644 --- a/freebsd/sys/sys/proc.h +++ b/freebsd/sys/sys/proc.h @@ -67,7 +67,7 @@ #include <sys/ucontext.h> #include <sys/ucred.h> #include <sys/types.h> -#include <sys/domainset.h> +#include <sys/_domainset.h> #include <machine/proc.h> /* Machine-dependent proc substruct. */ #ifdef _KERNEL @@ -328,6 +328,7 @@ struct thread { void *td_su; /* (k) FFS SU private */ sbintime_t td_sleeptimo; /* (t) Sleep timeout. */ int td_rtcgen; /* (s) rtc_generation of abs. sleep */ + size_t td_vslock_sz; /* (k) amount of vslock-ed space */ #define td_endzero td_sigmask /* Copied during fork1() or create_thread(). */ @@ -420,7 +421,11 @@ do { \ } while (0) #define TD_LOCKS_INC(td) ((td)->td_locks++) -#define TD_LOCKS_DEC(td) ((td)->td_locks--) +#define TD_LOCKS_DEC(td) do { \ + KASSERT(SCHEDULER_STOPPED_TD(td) || (td)->td_locks > 0, \ + ("thread %p owns no locks", (td))); \ + (td)->td_locks--; \ +} while (0) #else #define THREAD_LOCKPTR_ASSERT(td, lock) diff --git a/freebsd/sys/sys/random.h b/freebsd/sys/sys/random.h index 78acaf9d..69d377f5 100644 --- a/freebsd/sys/sys/random.h +++ b/freebsd/sys/sys/random.h @@ -31,10 +31,10 @@ #ifndef _SYS_RANDOM_H_ #define _SYS_RANDOM_H_ -#ifdef _KERNEL - #include <sys/types.h> +#ifdef _KERNEL + #if !defined(KLD_MODULE) #if defined(RANDOM_LOADABLE) && defined(RANDOM_YARROW) #error "Cannot define both RANDOM_LOADABLE and RANDOM_YARROW" @@ -137,4 +137,8 @@ void random_harvest_deregister_source(enum random_entropy_source); #endif /* _KERNEL */ +#define GRND_NONBLOCK 0x1 +#define GRND_RANDOM 0x2 +ssize_t getrandom(void *buf, size_t buflen, unsigned int flags); + #endif /* _SYS_RANDOM_H_ */ diff --git a/freebsd/sys/sys/refcount.h b/freebsd/sys/sys/refcount.h index e6b26ce4..040584ff 100644 --- a/freebsd/sys/sys/refcount.h +++ b/freebsd/sys/sys/refcount.h @@ -76,4 +76,35 @@ refcount_release(volatile u_int *count) return (1); } +/* + * A temporary hack until refcount_* APIs are sorted out. + */ +static __inline int +refcount_acquire_if_not_zero(volatile u_int *count) +{ + u_int old; + + old = *count; + for (;;) { + if (old == 0) + return (0); + if (atomic_fcmpset_int(count, &old, old + 1)) + return (1); + } +} + +static __inline int +refcount_release_if_not_last(volatile u_int *count) +{ + u_int old; + + old = *count; + for (;;) { + if (old == 1) + return (0); + if (atomic_fcmpset_int(count, &old, old - 1)) + return (1); + } +} + #endif /* ! __SYS_REFCOUNT_H__ */ diff --git a/freebsd/sys/sys/seq.h b/freebsd/sys/sys/seq.h index 405e3466..c5f00bcb 100644 --- a/freebsd/sys/sys/seq.h +++ b/freebsd/sys/sys/seq.h @@ -41,26 +41,52 @@ typedef uint32_t seq_t; #ifdef _KERNEL /* - * Typical usage: + * seq allows readers and writers to work with a consistent snapshot. Modifying + * operations must be enclosed within a transaction delineated by + * seq_write_beg/seq_write_end. The trick works by having the writer increment + * the sequence number twice, at the beginning and end of the transaction. + * The reader detects that the sequence number has not changed between its start + * and end, and that the sequence number is even, to validate consistency. + * + * Some fencing (both hard fencing and compiler barriers) may be needed, + * depending on the cpu. Modern AMD cpus provide strong enough guarantees to not + * require any fencing by the reader or writer. + * + * Example usage: * * writers: - * lock_exclusive(&obj->lock); - * seq_write_begin(&obj->seq); - * ..... - * seq_write_end(&obj->seq); - * unlock_exclusive(&obj->unlock); + * lock_exclusive(&obj->lock); + * seq_write_begin(&obj->seq); + * obj->var1 = ...; + * obj->var2 = ...; + * seq_write_end(&obj->seq); + * unlock_exclusive(&obj->lock); * * readers: - * obj_t lobj; - * seq_t seq; + * int var1, var2; + * seq_t seq; + * + * for (;;) { + * seq = seq_read(&obj->seq); + * var1 = obj->var1; + * var2 = obj->var2; + * if (seq_consistent(&obj->seq, seq)) + * break; + * } + * ..... + * + * Writers may not block or sleep in any way. + * + * There are 2 minor caveats in this implementation: + * + * 1. There is no guarantee of progress. That is, a large number of writers can + * interfere with the execution of the readers and cause the code to live-lock + * in a loop trying to acquire a consistent snapshot. * - * for (;;) { - * seq = seq_read(&gobj->seq); - * lobj = gobj; - * if (seq_consistent(&gobj->seq, seq)) - * break; - * } - * foo(lobj); + * 2. If the reader loops long enough, the counter may overflow and eventually + * wrap back to its initial value, fooling the reader into accepting the + * snapshot. Given that this needs 4 billion transactional writes across a + * single contended reader, it is unlikely to ever happen. */ /* A hack to get MPASS macro */ @@ -79,6 +105,7 @@ static __inline void seq_write_begin(seq_t *seqp) { + critical_enter(); MPASS(!seq_in_modify(*seqp)); *seqp += 1; atomic_thread_fence_rel(); @@ -90,6 +117,7 @@ seq_write_end(seq_t *seqp) atomic_store_rel_32(seqp, *seqp + 1); MPASS(!seq_in_modify(*seqp)); + critical_exit(); } static __inline seq_t diff --git a/freebsd/sys/sys/sysproto.h b/freebsd/sys/sys/sysproto.h index 9148a395..f70d820c 100644 --- a/freebsd/sys/sys/sysproto.h +++ b/freebsd/sys/sys/sysproto.h @@ -1815,6 +1815,11 @@ struct cpuset_setdomain_args { char mask_l_[PADL_(domainset_t *)]; domainset_t * mask; char mask_r_[PADR_(domainset_t *)]; char policy_l_[PADL_(int)]; int policy; char policy_r_[PADR_(int)]; }; +struct getrandom_args { + char buf_l_[PADL_(void *)]; void * buf; char buf_r_[PADR_(void *)]; + char buflen_l_[PADL_(size_t)]; size_t buflen; char buflen_r_[PADR_(size_t)]; + char flags_l_[PADL_(unsigned int)]; unsigned int flags; char flags_r_[PADR_(unsigned int)]; +}; int nosys(struct thread *, struct nosys_args *); void sys_sys_exit(struct thread *, struct sys_exit_args *); int sys_fork(struct thread *, struct fork_args *); @@ -2195,6 +2200,7 @@ int sys_mknodat(struct thread *, struct mknodat_args *); int sys_kevent(struct thread *, struct kevent_args *); int sys_cpuset_getdomain(struct thread *, struct cpuset_getdomain_args *); int sys_cpuset_setdomain(struct thread *, struct cpuset_setdomain_args *); +int sys_getrandom(struct thread *, struct getrandom_args *); #ifdef COMPAT_43 @@ -3087,6 +3093,7 @@ int freebsd11_mknodat(struct thread *, struct freebsd11_mknodat_args *); #define SYS_AUE_kevent AUE_KEVENT #define SYS_AUE_cpuset_getdomain AUE_NULL #define SYS_AUE_cpuset_setdomain AUE_NULL +#define SYS_AUE_getrandom AUE_NULL #endif /* __rtems__ */ #undef PAD_ diff --git a/freebsd/sys/sys/systm.h b/freebsd/sys/sys/systm.h index 2dfe959b..98efffd0 100644 --- a/freebsd/sys/sys/systm.h +++ b/freebsd/sys/sys/systm.h @@ -354,14 +354,15 @@ int copystr(const void * _Nonnull __restrict kfaddr, int copyinstr(const void * __restrict udaddr, void * _Nonnull __restrict kaddr, size_t len, size_t * __restrict lencopied); -int copyin(const void * _Nonnull __restrict udaddr, +int copyin(const void * __restrict udaddr, void * _Nonnull __restrict kaddr, size_t len); -int copyin_nofault(const void * _Nonnull __restrict udaddr, +int copyin_nofault(const void * __restrict udaddr, void * _Nonnull __restrict kaddr, size_t len); int copyout(const void * _Nonnull __restrict kaddr, - void * _Nonnull __restrict udaddr, size_t len); + void * __restrict udaddr, size_t len); int copyout_nofault(const void * _Nonnull __restrict kaddr, - void * _Nonnull __restrict udaddr, size_t len); + void * __restrict udaddr, size_t len); + #else /* __rtems__ */ static inline int copyinstr(const void * __restrict udaddr, void * __restrict kaddr, @@ -557,6 +558,8 @@ int pause_sbt(const char *wmesg, sbintime_t sbt, sbintime_t pr, #endif /* __rtems__ */ #define pause(wmesg, timo) \ pause_sbt((wmesg), tick_sbt * (timo), 0, C_HARDCLOCK) +#define pause_sig(wmesg, timo) \ + pause_sbt((wmesg), tick_sbt * (timo), 0, C_HARDCLOCK | C_CATCH) #define tsleep(chan, pri, wmesg, timo) \ _sleep((chan), NULL, (pri), (wmesg), tick_sbt * (timo), \ 0, C_HARDCLOCK) diff --git a/freebsd/sys/sys/vmmeter.h b/freebsd/sys/sys/vmmeter.h index 901604ae..408cd862 100644 --- a/freebsd/sys/sys/vmmeter.h +++ b/freebsd/sys/sys/vmmeter.h @@ -125,6 +125,7 @@ struct vmmeter { counter_u64_t v_vforkpages; /* (p) pages affected by vfork() */ counter_u64_t v_rforkpages; /* (p) pages affected by rfork() */ counter_u64_t v_kthreadpages; /* (p) ... and by kernel fork() */ + counter_u64_t v_wire_count; /* (p) pages wired down */ #define VM_METER_NCOUNTERS \ (offsetof(struct vmmeter, v_page_size) / sizeof(counter_u64_t)) /* @@ -139,24 +140,43 @@ struct vmmeter { u_int v_pageout_free_min; /* (c) min pages reserved for kernel */ u_int v_interrupt_free_min; /* (c) reserved pages for int code */ u_int v_free_severe; /* (c) severe page depletion point */ - u_int v_wire_count VMMETER_ALIGNED; /* (a) pages wired down */ - u_int v_active_count VMMETER_ALIGNED; /* (a) pages active */ - u_int v_inactive_count VMMETER_ALIGNED; /* (a) pages inactive */ - u_int v_laundry_count VMMETER_ALIGNED; /* (a) pages eligible for - laundering */ - u_int v_free_count VMMETER_ALIGNED; /* (f) pages free */ }; #endif /* _KERNEL || _WANT_VMMETER */ #ifdef _KERNEL +#include <sys/domainset.h> + extern struct vmmeter vm_cnt; -extern u_int vm_pageout_wakeup_thresh; +extern domainset_t vm_min_domains; +extern domainset_t vm_severe_domains; #define VM_CNT_ADD(var, x) counter_u64_add(vm_cnt.var, x) #define VM_CNT_INC(var) VM_CNT_ADD(var, 1) #define VM_CNT_FETCH(var) counter_u64_fetch(vm_cnt.var) +static inline void +vm_wire_add(int cnt) +{ + + VM_CNT_ADD(v_wire_count, cnt); +} + +static inline void +vm_wire_sub(int cnt) +{ + + VM_CNT_ADD(v_wire_count, -cnt); +} + +u_int vm_free_count(void); +static inline u_int +vm_wire_count(void) +{ + + return (VM_CNT_FETCH(v_wire_count)); +} + /* * Return TRUE if we are under our severe low-free-pages threshold * @@ -167,7 +187,7 @@ static inline int vm_page_count_severe(void) { - return (vm_cnt.v_free_severe > vm_cnt.v_free_count); + return (!DOMAINSET_EMPTY(&vm_severe_domains)); } /* @@ -183,50 +203,8 @@ static inline int vm_page_count_min(void) { - return (vm_cnt.v_free_min > vm_cnt.v_free_count); -} - -/* - * Return TRUE if we have not reached our free page target during - * free page recovery operations. - */ -static inline int -vm_page_count_target(void) -{ - - return (vm_cnt.v_free_target > vm_cnt.v_free_count); + return (!DOMAINSET_EMPTY(&vm_min_domains)); } -/* - * Return the number of pages we need to free-up or cache - * A positive number indicates that we do not have enough free pages. - */ -static inline int -vm_paging_target(void) -{ - - return (vm_cnt.v_free_target - vm_cnt.v_free_count); -} - -/* - * Returns TRUE if the pagedaemon needs to be woken up. - */ -static inline int -vm_paging_needed(u_int free_count) -{ - - return (free_count < vm_pageout_wakeup_thresh); -} - -/* - * Return the number of pages we need to launder. - * A positive number indicates that we have a shortfall of clean pages. - */ -static inline int -vm_laundry_target(void) -{ - - return (vm_paging_target()); -} #endif /* _KERNEL */ #endif /* _SYS_VMMETER_H_ */ diff --git a/freebsd/sys/vm/uma.h b/freebsd/sys/vm/uma.h index 2f80a448..935c5aa7 100644 --- a/freebsd/sys/vm/uma.h +++ b/freebsd/sys/vm/uma.h @@ -431,40 +431,6 @@ typedef void *(*uma_alloc)(uma_zone_t zone, vm_size_t size, int domain, typedef void (*uma_free)(void *item, vm_size_t size, uint8_t pflag); /* - * Sets up the uma allocator. (Called by vm_mem_init) - * - * Arguments: - * bootmem A pointer to memory used to bootstrap the system. - * - * Returns: - * Nothing - * - * Discussion: - * This memory is used for zones which allocate things before the - * backend page supplier can give us pages. It should be - * UMA_SLAB_SIZE * boot_pages bytes. (see uma_int.h) - * - */ - -void uma_startup(void *bootmem, int boot_pages); - -/* - * Finishes starting up the allocator. This should - * be called when kva is ready for normal allocs. - * - * Arguments: - * None - * - * Returns: - * Nothing - * - * Discussion: - * uma_startup2 is called by kmeminit() to enable us of uma for malloc. - */ - -void uma_startup2(void); - -/* * Reclaims unused memory for all zones * * Arguments: diff --git a/freebsd/sys/vm/uma_core.c b/freebsd/sys/vm/uma_core.c index a1c45c93..fdf7dc35 100644 --- a/freebsd/sys/vm/uma_core.c +++ b/freebsd/sys/vm/uma_core.c @@ -165,13 +165,10 @@ static struct rwlock_padalign __exclusive_cache_line uma_rwlock; #ifndef __rtems__ /* * Pointer and counter to pool of pages, that is preallocated at - * startup to bootstrap UMA. Early zones continue to use the pool - * until it is depleted, so allocations may happen after boot, thus - * we need a mutex to protect it. + * startup to bootstrap UMA. */ static char *bootmem; static int boot_pages; -static struct mtx uma_boot_pages_mtx; #endif /* __rtems__ */ static struct sx uma_drain_lock; @@ -182,9 +179,8 @@ static volatile unsigned long uma_kmem_total; #ifndef __rtems__ /* Is the VM done starting up? */ -static int booted = 0; -#define UMA_STARTUP 1 -#define UMA_STARTUP2 2 +static enum { BOOT_COLD = 0, BOOT_STRAPPED, BOOT_PAGEALLOC, BOOT_BUCKETS, + BOOT_RUNNING } booted = BOOT_COLD; #endif /* __rtems__ */ /* @@ -267,6 +263,15 @@ enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI }; /* Prototypes.. */ #ifndef __rtems__ +int uma_startup_count(int); +#endif /* __rtems__ */ +void uma_startup(void *, int); +#ifndef __rtems__ +void uma_startup1(void); +void uma_startup2(void); +#endif /* __rtems__ */ + +#ifndef __rtems__ static void *noobj_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); #endif /* __rtems__ */ static void *page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); @@ -1132,33 +1137,46 @@ startup_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, int pages; keg = zone_first_keg(zone); - pages = howmany(bytes, PAGE_SIZE); - KASSERT(pages > 0, ("startup_alloc can't reserve 0 pages\n")); /* - * Check our small startup cache to see if it has pages remaining. - */ - mtx_lock(&uma_boot_pages_mtx); - if (pages <= boot_pages) { - mem = bootmem; - boot_pages -= pages; - bootmem += pages * PAGE_SIZE; - mtx_unlock(&uma_boot_pages_mtx); - *pflag = UMA_SLAB_BOOT; - return (mem); - } - mtx_unlock(&uma_boot_pages_mtx); - if (booted < UMA_STARTUP2) - panic("UMA: Increase vm.boot_pages"); - /* - * Now that we've booted reset these users to their real allocator. + * If we are in BOOT_BUCKETS or higher, than switch to real + * allocator. Zones with page sized slabs switch at BOOT_PAGEALLOC. */ + switch (booted) { + case BOOT_COLD: + case BOOT_STRAPPED: + break; + case BOOT_PAGEALLOC: + if (keg->uk_ppera > 1) + break; + case BOOT_BUCKETS: + case BOOT_RUNNING: #ifdef UMA_MD_SMALL_ALLOC - keg->uk_allocf = (keg->uk_ppera > 1) ? page_alloc : uma_small_alloc; + keg->uk_allocf = (keg->uk_ppera > 1) ? + page_alloc : uma_small_alloc; #else - keg->uk_allocf = page_alloc; + keg->uk_allocf = page_alloc; #endif - return keg->uk_allocf(zone, bytes, domain, pflag, wait); + return keg->uk_allocf(zone, bytes, domain, pflag, wait); + } + + /* + * Check our small startup cache to see if it has pages remaining. + */ + pages = howmany(bytes, PAGE_SIZE); + KASSERT(pages > 0, ("%s can't reserve 0 pages", __func__)); + if (pages > boot_pages) + panic("UMA zone \"%s\": Increase vm.boot_pages", zone->uz_name); +#ifdef DIAGNOSTIC + printf("%s from \"%s\", %d boot pages left\n", __func__, zone->uz_name, + boot_pages); +#endif + mem = bootmem; + boot_pages -= pages; + bootmem += pages * PAGE_SIZE; + *pflag = UMA_SLAB_BOOT; + + return (mem); } #endif /* __rtems__ */ @@ -1543,7 +1561,7 @@ keg_ctor(void *mem, int size, void *udata, int flags) * If we haven't booted yet we need allocations to go through the * startup cache until the vm is ready. */ - if (booted < UMA_STARTUP2) + if (booted < BOOT_PAGEALLOC) keg->uk_allocf = startup_alloc; #ifdef UMA_MD_SMALL_ALLOC else if (keg->uk_ppera == 1) @@ -1844,33 +1862,95 @@ zone_foreach(void (*zfunc)(uma_zone_t)) rw_runlock(&uma_rwlock); } -/* Public functions */ -/* See uma.h */ +#ifndef __rtems__ +/* + * Count how many pages do we need to bootstrap. VM supplies + * its need in early zones in the argument, we add up our zones, + * which consist of: UMA Slabs, UMA Hash and 9 Bucket zones. The + * zone of zones and zone of kegs are accounted separately. + */ +#define UMA_BOOT_ZONES 11 +#endif /* __rtems__ */ +/* Zone of zones and zone of kegs have arbitrary alignment. */ +#define UMA_BOOT_ALIGN 32 +#ifndef __rtems__ +static int zsize, ksize; +int +uma_startup_count(int vm_zones) +{ + int zones, pages; + + ksize = sizeof(struct uma_keg) + + (sizeof(struct uma_domain) * vm_ndomains); + zsize = sizeof(struct uma_zone) + + (sizeof(struct uma_cache) * (mp_maxid + 1)) + + (sizeof(struct uma_zone_domain) * vm_ndomains); + + /* + * Memory for the zone of kegs and its keg, + * and for zone of zones. + */ + pages = howmany(roundup(zsize, CACHE_LINE_SIZE) * 2 + + roundup(ksize, CACHE_LINE_SIZE), PAGE_SIZE); + +#ifdef UMA_MD_SMALL_ALLOC + zones = UMA_BOOT_ZONES; +#else + zones = UMA_BOOT_ZONES + vm_zones; + vm_zones = 0; +#endif + + /* Memory for the rest of startup zones, UMA and VM, ... */ + if (zsize > UMA_SLAB_SIZE) + pages += (zones + vm_zones) * + howmany(roundup2(zsize, UMA_BOOT_ALIGN), UMA_SLAB_SIZE); + else + pages += howmany(zones, + UMA_SLAB_SPACE / roundup2(zsize, UMA_BOOT_ALIGN)); + + /* ... and their kegs. Note that zone of zones allocates a keg! */ + pages += howmany(zones + 1, + UMA_SLAB_SPACE / roundup2(ksize, UMA_BOOT_ALIGN)); + + /* + * Most of startup zones are not going to be offpages, that's + * why we use UMA_SLAB_SPACE instead of UMA_SLAB_SIZE in all + * calculations. Some large bucket zones will be offpage, and + * thus will allocate hashes. We take conservative approach + * and assume that all zones may allocate hash. This may give + * us some positive inaccuracy, usually an extra single page. + */ + pages += howmany(zones, UMA_SLAB_SPACE / + (sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT)); + + return (pages); +} +#endif /* __rtems__ */ + void uma_startup(void *mem, int npages) { struct uma_zctor_args args; uma_keg_t masterkeg; uintptr_t m; -#ifndef __rtems__ - int zsize; - int ksize; -#else /* __rtems__ */ +#ifdef __rtems__ size_t zsize, ksize, size; -#endif /* __rtems__ */ - - rw_init(&uma_rwlock, "UMA lock"); ksize = sizeof(struct uma_keg) + (sizeof(struct uma_domain) * vm_ndomains); zsize = sizeof(struct uma_zone) + - (sizeof(struct uma_cache) * mp_ncpus) + + (sizeof(struct uma_cache) * (mp_maxid + 1)) + (sizeof(struct uma_zone_domain) * vm_ndomains); -#ifdef __rtems__ size = 2 * roundup(zsize, CACHE_LINE_SIZE) + roundup(ksize, CACHE_LINE_SIZE); #endif /* __rtems__ */ +#ifdef DIAGNOSTIC + printf("Entering %s with %d boot pages configured\n", __func__, npages); +#endif + + rw_init(&uma_rwlock, "UMA lock"); + #ifndef __rtems__ /* Use bootpages memory for the zone of zones and zone of kegs. */ m = (uintptr_t)mem; @@ -1901,26 +1981,23 @@ uma_startup(void *mem, int npages) args.uminit = zero_init; args.fini = NULL; args.keg = masterkeg; - args.align = 32 - 1; + args.align = UMA_BOOT_ALIGN - 1; args.flags = UMA_ZFLAG_INTERNAL; zone_ctor(kegs, zsize, &args, M_WAITOK); #ifndef __rtems__ - mtx_init(&uma_boot_pages_mtx, "UMA boot pages", NULL, MTX_DEF); bootmem = mem; boot_pages = npages; #endif /* __rtems__ */ args.name = "UMA Zones"; - args.size = sizeof(struct uma_zone) + - (sizeof(struct uma_cache) * (mp_maxid + 1)) + - (sizeof(struct uma_zone_domain) * vm_ndomains); + args.size = zsize; args.ctor = zone_ctor; args.dtor = zone_dtor; args.uminit = zero_init; args.fini = NULL; args.keg = NULL; - args.align = 32 - 1; + args.align = UMA_BOOT_ALIGN - 1; args.flags = UMA_ZFLAG_INTERNAL; zone_ctor(zones, zsize, &args, M_WAITOK); @@ -1938,33 +2015,31 @@ uma_startup(void *mem, int npages) bucket_init(); #ifndef __rtems__ - booted = UMA_STARTUP; + booted = BOOT_STRAPPED; #endif /* __rtems__ */ } -#ifdef __rtems__ -static void -rtems_bsd_uma_startup(void *unused) + +#ifndef __rtems__ +void +uma_startup1(void) { - (void) unused; - uma_kmem_limit = (rtems_bsd_get_allocator_domain_size( - RTEMS_BSD_ALLOCATOR_DOMAIN_PAGE) / 4) * 3; - sx_init_flags(&uma_drain_lock, "umadrain", SX_RECURSE); - uma_startup(NULL, 0); +#ifdef DIAGNOSTIC + printf("Entering %s with %d boot pages left\n", __func__, boot_pages); +#endif + booted = BOOT_PAGEALLOC; } -SYSINIT(rtems_bsd_uma_startup, SI_SUB_VM, SI_ORDER_SECOND, - rtems_bsd_uma_startup, NULL); -#endif /* __rtems__ */ - -#ifndef __rtems__ -/* see uma.h */ void uma_startup2(void) { - booted = UMA_STARTUP2; - bucket_enable(); + +#ifdef DIAGNOSTIC + printf("Entering %s with %d boot pages left\n", __func__, boot_pages); +#endif + booted = BOOT_BUCKETS; sx_init(&uma_drain_lock, "umadrain"); + bucket_enable(); } #endif /* __rtems__ */ @@ -1972,11 +2047,13 @@ uma_startup2(void) * Initialize our callout handle * */ - static void uma_startup3(void) { +#ifndef __rtems__ + booted = BOOT_RUNNING; +#endif /* __rtems__ */ callout_init(&uma_callout, 1); callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); } @@ -1996,6 +2073,7 @@ uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini, return (zone_alloc_item(kegs, &args, UMA_ANYDOMAIN, M_WAITOK)); } +/* Public functions */ /* See uma.h */ void uma_set_align(int align) @@ -2047,7 +2125,7 @@ uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor, args.keg = NULL; #ifndef __rtems__ - if (booted < UMA_STARTUP2) { + if (booted < BOOT_BUCKETS) { locked = false; } else { #endif /* __rtems__ */ @@ -2089,7 +2167,7 @@ uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor, args.keg = keg; #ifndef __rtems__ - if (booted < UMA_STARTUP2) { + if (booted < BOOT_BUCKETS) { locked = false; } else { #endif /* __rtems__ */ @@ -2735,7 +2813,9 @@ zone_import(uma_zone_t zone, void **bucket, int max, int domain, int flags) { uma_slab_t slab; uma_keg_t keg; +#ifndef __rtems__ int stripe; +#endif /* __rtems__ */ int i; slab = NULL; @@ -2745,7 +2825,9 @@ zone_import(uma_zone_t zone, void **bucket, int max, int domain, int flags) if ((slab = zone->uz_slab(zone, keg, domain, flags)) == NULL) break; keg = slab->us_keg; +#ifndef __rtems__ stripe = howmany(max, vm_ndomains); +#endif /* __rtems__ */ while (slab->us_freecount && i < max) { bucket[i++] = slab_alloc_item(keg, slab); if (keg->uk_free <= keg->uk_reserve) @@ -3561,7 +3643,7 @@ uma_large_malloc_domain(vm_size_t size, int domain, int wait) slab->us_data = (void *)addr; slab->us_flags = UMA_SLAB_KERNEL | UMA_SLAB_MALLOC; slab->us_size = size; - slab->us_domain = vm_phys_domidx(PHYS_TO_VM_PAGE( + slab->us_domain = vm_phys_domain(PHYS_TO_VM_PAGE( pmap_kextract(addr))); uma_total_inc(size); } else { @@ -3767,7 +3849,7 @@ sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS) { struct uma_stream_header ush; struct uma_type_header uth; - struct uma_percpu_stat ups; + struct uma_percpu_stat *ups; uma_bucket_t bucket; uma_zone_domain_t zdom; struct sbuf sbuf; @@ -3783,6 +3865,7 @@ sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS) return (error); sbuf_new_for_sysctl(&sbuf, NULL, 128, req); sbuf_clear_flags(&sbuf, SBUF_INCLUDENUL); + ups = malloc((mp_maxid + 1) * sizeof(*ups), M_TEMP, M_WAITOK); count = 0; rw_rlock(&uma_rwlock); @@ -3835,7 +3918,6 @@ sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS) uth.uth_frees = z->uz_frees; uth.uth_fails = z->uz_fails; uth.uth_sleeps = z->uz_sleeps; - (void)sbuf_bcat(&sbuf, &uth, sizeof(uth)); /* * While it is not normally safe to access the cache * bucket pointers while not on the CPU that owns the @@ -3844,30 +3926,31 @@ sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS) * accept the possible race associated with bucket * exchange during monitoring. */ - for (i = 0; i < (mp_maxid + 1); i++) { - bzero(&ups, sizeof(ups)); - if (kz->uk_flags & UMA_ZFLAG_INTERNAL) - goto skip; - if (CPU_ABSENT(i)) - goto skip; + for (i = 0; i < mp_maxid + 1; i++) { + bzero(&ups[i], sizeof(*ups)); + if (kz->uk_flags & UMA_ZFLAG_INTERNAL || + CPU_ABSENT(i)) + continue; cache = &z->uz_cpu[i]; if (cache->uc_allocbucket != NULL) - ups.ups_cache_free += + ups[i].ups_cache_free += cache->uc_allocbucket->ub_cnt; if (cache->uc_freebucket != NULL) - ups.ups_cache_free += + ups[i].ups_cache_free += cache->uc_freebucket->ub_cnt; - ups.ups_allocs = cache->uc_allocs; - ups.ups_frees = cache->uc_frees; -skip: - (void)sbuf_bcat(&sbuf, &ups, sizeof(ups)); + ups[i].ups_allocs = cache->uc_allocs; + ups[i].ups_frees = cache->uc_frees; } ZONE_UNLOCK(z); + (void)sbuf_bcat(&sbuf, &uth, sizeof(uth)); + for (i = 0; i < mp_maxid + 1; i++) + (void)sbuf_bcat(&sbuf, &ups[i], sizeof(ups[i])); } } rw_runlock(&uma_rwlock); error = sbuf_finish(&sbuf); sbuf_delete(&sbuf); + free(ups, M_TEMP); return (error); } @@ -4063,6 +4146,20 @@ DB_SHOW_COMMAND(umacache, db_show_umacache) #endif /* DDB */ #endif /* __rtems__ */ #ifdef __rtems__ +static void +rtems_bsd_uma_startup(void *unused) +{ + (void) unused; + + uma_kmem_limit = rtems_bsd_get_allocator_domain_size( + RTEMS_BSD_ALLOCATOR_DOMAIN_PAGE); + sx_init_flags(&uma_drain_lock, "umadrain", SX_RECURSE); + uma_startup(NULL, 0); +} + +SYSINIT(rtems_bsd_uma_startup, SI_SUB_VM, SI_ORDER_SECOND, + rtems_bsd_uma_startup, NULL); + /* * This is a helper routine for test programs. The uma_timeout() may need some * dynamic memory. This could disturb out of memory tests. diff --git a/freebsd/sys/vm/uma_int.h b/freebsd/sys/vm/uma_int.h index 5852a1c3..53d65dac 100644 --- a/freebsd/sys/vm/uma_int.h +++ b/freebsd/sys/vm/uma_int.h @@ -134,14 +134,15 @@ #define UMA_SLAB_MASK (PAGE_SIZE - 1) /* Mask to get back to the page */ #define UMA_SLAB_SHIFT PAGE_SHIFT /* Number of bits PAGE_MASK */ -#define UMA_BOOT_PAGES 64 /* Pages allocated for startup */ -#define UMA_BOOT_PAGES_ZONES 32 /* Multiplier for pages to reserve */ - /* if uma_zone > PAGE_SIZE */ - /* Max waste percentage before going to off page slab management */ #define UMA_MAX_WASTE 10 /* + * Size of memory in a not offpage slab available for actual items. + */ +#define UMA_SLAB_SPACE (UMA_SLAB_SIZE - sizeof(struct uma_slab)) + +/* * I doubt there will be many cases where this is exceeded. This is the initial * size of the hash table for uma_slabs that are managed off page. This hash * does expand by powers of two. Currently it doesn't get smaller. diff --git a/freebsd/sys/vm/vm.h b/freebsd/sys/vm/vm.h index b5de53d3..94809bad 100644 --- a/freebsd/sys/vm/vm.h +++ b/freebsd/sys/vm/vm.h @@ -80,7 +80,9 @@ typedef u_char vm_prot_t; /* protection codes */ #define VM_PROT_WRITE ((vm_prot_t) 0x02) #define VM_PROT_EXECUTE ((vm_prot_t) 0x04) #define VM_PROT_COPY ((vm_prot_t) 0x08) /* copy-on-read */ -#define VM_PROT_FAULT_LOOKUP ((vm_prot_t) 0x010) +#define VM_PROT_PRIV_FLAG ((vm_prot_t) 0x10) +#define VM_PROT_FAULT_LOOKUP VM_PROT_PRIV_FLAG +#define VM_PROT_QUICK_NOFAULT VM_PROT_PRIV_FLAG /* same to save bits */ #define VM_PROT_ALL (VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE) #define VM_PROT_RW (VM_PROT_READ|VM_PROT_WRITE) @@ -146,7 +148,11 @@ extern void vm_ksubmap_init(struct kva_md_info *); extern int old_mlock; +#ifndef __rtems__ +extern int vm_ndomains; +#else /* __rtems__ */ #define vm_ndomains 1 +#endif /* __rtems__ */ struct ucred; int swap_reserve(vm_ooffset_t incr); diff --git a/freebsd/sys/vm/vm_extern.h b/freebsd/sys/vm/vm_extern.h index e64637c1..47e35b2d 100644 --- a/freebsd/sys/vm/vm_extern.h +++ b/freebsd/sys/vm/vm_extern.h @@ -144,5 +144,9 @@ struct sf_buf *vm_imgact_map_page(vm_object_t object, vm_ooffset_t offset); void vm_imgact_unmap_page(struct sf_buf *sf); void vm_thread_dispose(struct thread *td); int vm_thread_new(struct thread *td, int pages); +u_int vm_active_count(void); +u_int vm_inactive_count(void); +u_int vm_laundry_count(void); +u_int vm_wait_count(void); #endif /* _KERNEL */ #endif /* !_VM_EXTERN_H_ */ |