 @@ -1,437 +1,444 @@
-/*	$NetBSD: npf_alg_icmp.c,v 1.19 2014/02/16 22:10:40 rmind Exp $	*/
+/*	$NetBSD: npf_alg_icmp.c,v 1.20 2014/02/19 03:51:31 rmind Exp $	*/
 /*-
  * Copyright (c) 2010 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This material is based upon work partially supported by The
  * NetBSD Foundation under a contract with Mindaugas Rasiukevicius.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ *
  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 /*
  * NPF ALG for ICMP and traceroute translations.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: npf_alg_icmp.c,v 1.19 2014/02/16 22:10:40 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: npf_alg_icmp.c,v 1.20 2014/02/19 03:51:31 rmind Exp $");
 #include <sys/param.h>
 #include <sys/module.h>
 #include <netinet/in_systm.h>
 #include <netinet/in.h>
 #include <netinet/ip.h>
 #include <netinet/tcp.h>
 #include <netinet/udp.h>
 #include <netinet/ip_icmp.h>
 #include <netinet/icmp6.h>
 #include <net/pfil.h>
 #include "npf_impl.h"
 MODULE(MODULE_CLASS_MISC, npf_alg_icmp, "npf");
 /*
  * Traceroute criteria.
+ *
  * IANA assigned base port: 33434.  However, common practice is to increase
  * the port, thus monitor [33434-33484] range.  Additional filter is low TTL.
  */
 #define	TR_BASE_PORT	33434
 #define	TR_PORT_RANGE	33484
 #define	TR_MAX_TTL	48
 static npf_alg_t *	alg_icmp	__read_mostly;
 /*
  * npfa_icmp_match: matching insperctor determines ALG case and associates
  * our ALG with the NAT entry.
  */
 static bool
 npfa_icmp_match(npf_cache_t *npc, nbuf_t *nbuf, npf_nat_t *nt, int di)
+{
 	const int proto = npc->npc_proto;
 	const struct ip *ip = npc->npc_ip.v4;
 	in_port_t dport;
 	KASSERT(npf_iscached(npc, NPC_IP46));
 	KASSERT(npf_iscached(npc, NPC_LAYER4));
 	/* Check for low TTL.  Also, we support outbound NAT only. */
 	if (ip->ip_ttl > TR_MAX_TTL || di != PFIL_OUT) {
 		return false;
+	}
 	switch (proto) {
 	case IPPROTO_TCP: {
 		const struct tcphdr *th = npc->npc_l4.tcp;
 		dport = ntohs(th->th_dport);
 		break;
+	}
 	case IPPROTO_UDP: {
 		const struct udphdr *uh = npc->npc_l4.udp;
 		dport = ntohs(uh->uh_dport);
 		break;
+	}
 	case IPPROTO_ICMP:
 	case IPPROTO_ICMPV6:
 		/* Just to pass the test below. */
 		dport = TR_BASE_PORT;
 		break;
 	default:
 		return false;
+	}
 	/* Handle TCP/UDP traceroute - check for port range. */
 	if (dport < TR_BASE_PORT || dport > TR_PORT_RANGE) {
 		return false;
+	}
 	/* Associate ALG with translation entry. */
 	npf_nat_setalg(nt, alg_icmp, 0);
 	return true;
+}
 /*
  * npfa_icmp{4,6}_inspect: retrieve unique identifiers - either ICMP query
  * ID or TCP/UDP ports of the original packet, which is embedded.
  */
 static bool
 npfa_icmp4_inspect(const int type, npf_cache_t *npc, nbuf_t *nbuf)
+{
 	u_int offby;
 	/* Per RFC 792. */
 	switch (type) {
 	case ICMP_UNREACH:
 	case ICMP_SOURCEQUENCH:
 	case ICMP_REDIRECT:
 	case ICMP_TIMXCEED:
 	case ICMP_PARAMPROB:
 		if (npc == NULL) {
 			return false;
+		}
 		/* Should contain original IP header. */
 		if (!nbuf_advance(nbuf, offsetof(struct icmp, icmp_ip), 0)) {
 			return false;
+		}
 		return (npf_cache_all(npc, nbuf) & NPC_LAYER4) != 0;
 	case ICMP_ECHOREPLY:
 	case ICMP_ECHO:
 	case ICMP_TSTAMP:
 	case ICMP_TSTAMPREPLY:
 	case ICMP_IREQ:
 	case ICMP_IREQREPLY:
 		/* Should contain ICMP query ID - ensure. */
 		offby = offsetof(struct icmp, icmp_id);
 		if (!nbuf_advance(nbuf, offby, sizeof(uint16_t))) {
 			return false;
+		}
 		npc->npc_info |= NPC_ICMP_ID;
 		return true;
 	default:
 		break;
+	}
 	return false;
+}
 static bool
 npfa_icmp6_inspect(const int type, npf_cache_t *npc, nbuf_t *nbuf)
+{
 	u_int offby;
 	/* Per RFC 4443. */
 	switch (type) {
 	case ICMP6_DST_UNREACH:
 	case ICMP6_PACKET_TOO_BIG:
 	case ICMP6_TIME_EXCEEDED:
 	case ICMP6_PARAM_PROB:
 		if (npc == NULL) {
 			return false;
+		}
 		/* Should contain original IP header. */
 		if (!nbuf_advance(nbuf, sizeof(struct icmp6_hdr), 0)) {
 			return false;
+		}
 		return (npf_cache_all(npc, nbuf) & NPC_LAYER4) != 0;
 	case ICMP6_ECHO_REQUEST:
 	case ICMP6_ECHO_REPLY:
 		/* Should contain ICMP query ID - ensure. */
 		offby = offsetof(struct icmp6_hdr, icmp6_id);
 		if (!nbuf_advance(nbuf, offby, sizeof(uint16_t))) {
 			return false;
+		}
 		npc->npc_info |= NPC_ICMP_ID;
 		return true;
 	default:
 		break;
+	}
 	return false;
+}
 /*
  * npfa_icmp_session: ALG ICMP inspector.
+ *
  * => Returns true if "enpc" is filled.
  */
 static bool
 npfa_icmp_inspect(npf_cache_t *npc, nbuf_t *nbuf, npf_cache_t *enpc)
+{
 	bool ret;
 	KASSERT(npf_iscached(npc, NPC_IP46));
 	KASSERT(npf_iscached(npc, NPC_ICMP));
 	/* Advance to ICMP header. */
 	nbuf_reset(nbuf);
 	if (!nbuf_advance(nbuf, npc->npc_hlen, 0)) {
 		return false;
+	}
 	enpc->npc_info = 0;
 	/*
 	 * Inspect the ICMP packet.  The relevant data might be in the
 	 * embedded packet.  Fill the "enpc" cache, if so.
 	 */
 	if (npf_iscached(npc, NPC_IP4)) {
 		const struct icmp *ic = npc->npc_l4.icmp;
 		ret = npfa_icmp4_inspect(ic->icmp_type, enpc, nbuf);
 	} else if (npf_iscached(npc, NPC_IP6)) {
 		const struct icmp6_hdr *ic6 = npc->npc_l4.icmp6;
 		ret = npfa_icmp6_inspect(ic6->icmp6_type, enpc, nbuf);
 	} else {
 		ret = false;
+	}
 	if (!ret) {
 		return false;
+	}
 	/* ICMP ID is the original packet, just indicate it. */
 	if (npf_iscached(enpc, NPC_ICMP_ID)) {
 		npc->npc_info |= NPC_ICMP_ID;
 		return false;
+	}
 	/* Indicate that embedded packet is in the cache. */
 	return true;
+}
 static npf_session_t *
 npfa_icmp_session(npf_cache_t *npc, nbuf_t *nbuf, int di)
+{
 	npf_cache_t enpc;
 	/* Inspect ICMP packet for an embedded packet. */
 	if (!npf_iscached(npc, NPC_ICMP))
 		return NULL;
 	if (!npfa_icmp_inspect(npc, nbuf, &enpc))
 		return NULL;
 	/*
 	 * Invert the identifiers of the embedded packet.
 	 * If it is ICMP, then ensure ICMP ID.
 	 */
 	union l4 {
 		struct tcphdr th;
 		struct udphdr uh;
 	} l4;
 	bool ret, forw;
 	#define	SWAP(type, x, y) { type tmp = x; x = y; y = tmp; }
 	SWAP(npf_addr_t *, enpc.npc_ips[NPF_SRC], enpc.npc_ips[NPF_DST]);
 	switch (enpc.npc_proto) {
 	case IPPROTO_TCP:
 		l4.th.th_sport = enpc.npc_l4.tcp->th_dport;
 		l4.th.th_dport = enpc.npc_l4.tcp->th_sport;
 		enpc.npc_l4.tcp = &l4.th;
 		break;
 	case IPPROTO_UDP:
 		l4.uh.uh_sport = enpc.npc_l4.udp->uh_dport;
 		l4.uh.uh_dport = enpc.npc_l4.udp->uh_sport;
 		enpc.npc_l4.udp = &l4.uh;
 		break;
 	case IPPROTO_ICMP: {
 		const struct icmp *ic = enpc.npc_l4.icmp;
 		ret = npfa_icmp4_inspect(ic->icmp_type, &enpc, nbuf);
 		if (!ret || !npf_iscached(&enpc, NPC_ICMP_ID))
 			return false;
 		break;
+	}
 	case IPPROTO_ICMPV6: {
 		const struct icmp6_hdr *ic6 = enpc.npc_l4.icmp6;
 		ret = npfa_icmp6_inspect(ic6->icmp6_type, &enpc, nbuf);
 		if (!ret || !npf_iscached(&enpc, NPC_ICMP_ID))
 			return false;
 		break;
+	}
 	default:
 		return false;
+	}
 	/* Lookup for a session using embedded packet. */
 	return npf_session_lookup(&enpc, nbuf, di, &forw);
+}
 /*
  * npfa_icmp_nat: ALG translator - rewrites IP address in the IP header
  * which is embedded in ICMP packet.  Note: backwards stream only.
  */
 static bool
 npfa_icmp_nat(npf_cache_t *npc, nbuf_t *nbuf, npf_nat_t *nt, bool forw)
+{
 	const u_int which = NPF_SRC;
 	npf_cache_t enpc;
 	if (forw || !npf_iscached(npc, NPC_ICMP))
 		return false;
 	if (!npfa_icmp_inspect(npc, nbuf, &enpc))
 		return false;
 	KASSERT(npf_iscached(&enpc, NPC_IP46));
 	KASSERT(npf_iscached(&enpc, NPC_LAYER4));
 	/*
 	 * ICMP: fetch the current checksum we are going to fixup.
 	 */
 	struct icmp *ic = npc->npc_l4.icmp;
 	uint16_t cksum = ic->icmp_cksum;
 	CTASSERT(offsetof(struct icmp, icmp_cksum) ==
 	    offsetof(struct icmp6_hdr, icmp6_cksum));
 	/*
-	 * Retrieve the original address and port, then calculate ICMP
+	 * Fetch the IP and port in the _embedded_ packet.  Also, fetch
-	 * checksum for these changes in the embedded packet.  While data
+	 * the IPv4 and TCP/UDP checksums before they are rewritten.
-	 * is not rewritten in the cache, save IP and TCP/UDP checksums.
+	 * Calculate the part of the ICMP checksum fixup.
 	 * XXX: Assumes NPF_NATOUT (source address/port).  Currently,
 	 * npfa_icmp_match() matches only for the PFIL_OUT traffic.
 	 */
 	const int proto = enpc.npc_proto;
 	uint16_t ipcksum = 0, l4cksum = 0;
 	npf_addr_t *addr;
 	in_port_t port;
 	npf_nat_getorig(nt, &addr, &port);
 	if (npf_iscached(&enpc, NPC_IP4)) {
 		const struct ip *eip = enpc.npc_ip.v4;
 		ipcksum = eip->ip_sum;
+	}
-	cksum = npf_addr_cksum(cksum, enpc.npc_alen, enpc.npc_ips[NPF_SRC], addr);
+	cksum = npf_addr_cksum(cksum, enpc.npc_alen, enpc.npc_ips[which], addr);
 	switch (proto) {
 	case IPPROTO_TCP: {
 		const struct tcphdr *th = enpc.npc_l4.tcp;
 		cksum = npf_fixup16_cksum(cksum, th->th_sport, port);
 		l4cksum = th->th_sum;
 		break;
+	}
 	case IPPROTO_UDP: {
 		const struct udphdr *uh = enpc.npc_l4.udp;
 		cksum = npf_fixup16_cksum(cksum, uh->uh_sport, port);
 		l4cksum = uh->uh_sum;
 		break;
+	}
 	case IPPROTO_ICMP:
 	case IPPROTO_ICMPV6:
 		break;
 	default:
 		return false;
+	}
 	/*
-	 * Rewrite the source IP address and port of the embedded IP header,
+	 * Translate the embedded packet.  The following changes will
-	 * which represents the original packet.  This updates the checksums
+	 * be performed by npf_napt_rwr():
 	 * in the embedded packet.
 	 *	1) Rewrite the IP address and, if not ICMP, port.
 	 *	2) Rewrite the TCP/UDP checksum (if not ICMP).
 	 *	3) Rewrite the IPv4 checksum for (1) and (2).
+	 *
 	 * XXX: Assumes NPF_NATOUT (source address/port).  Currently,
 	 * npfa_icmp_match() matches only for the PFIL_OUT traffic.
 	 */
-	if (npf_nat_translate(&enpc, nbuf, nt, forw)) {
+	if (npf_napt_rwr(&enpc, which, addr, port)) {
 		return false;
+	}
 	/*
-	 * Finish calculation of the ICMP checksum: include the checksum
+	 * Finally, finish the ICMP checksum fixup: include the checksum
-	 * change in the embedded packet.
+	 * changes in the embedded packet.
 	 */
 	if (npf_iscached(&enpc, NPC_IP4)) {
 		const struct ip *eip = enpc.npc_ip.v4;
 		cksum = npf_fixup16_cksum(cksum, ipcksum, eip->ip_sum);
+	}
 	switch (proto) {
 	case IPPROTO_TCP: {
 		const struct tcphdr *th = enpc.npc_l4.tcp;
 		cksum = npf_fixup16_cksum(cksum, l4cksum, th->th_sum);
 		break;
+	}
 	case IPPROTO_UDP:
 		if (l4cksum) {
 			const struct udphdr *uh = enpc.npc_l4.udp;
 			cksum = npf_fixup16_cksum(cksum, l4cksum, uh->uh_sum);
+		}
 		break;
+	}
 	ic->icmp_cksum = cksum;
 	return true;
+}
 /*
  * npf_alg_icmp_{init,fini,modcmd}: ICMP ALG initialization, destruction
  * and module interface.
  */
 static int
 npf_alg_icmp_init(void)
+{
 	static const npfa_funcs_t icmp = {
 		.match		= npfa_icmp_match,
 		.translate	= npfa_icmp_nat,
 		.inspect	= npfa_icmp_session,
 	};
 	alg_icmp = npf_alg_register("icmp", &icmp);
 	return alg_icmp ? 0 : ENOMEM;
+}
 static int
 npf_alg_icmp_fini(void)
+{
 	KASSERT(alg_icmp != NULL);
 	return npf_alg_unregister(alg_icmp);
+}
 static int
 npf_alg_icmp_modcmd(modcmd_t cmd, void *arg)
+{
 	switch (cmd) {
 	case MODULE_CMD_INIT:
 		return npf_alg_icmp_init();
 	case MODULE_CMD_FINI:
 		return npf_alg_icmp_fini();
 	case MODULE_CMD_AUTOUNLOAD:
 		return EBUSY;
 	default:
 		return ENOTTY;
+	}
 	return 0;
+}

 @@ -1,370 +1,371 @@
-/*	$NetBSD: npf_impl.h,v 1.48 2014/02/16 22:10:40 rmind Exp $	*/
+/*	$NetBSD: npf_impl.h,v 1.49 2014/02/19 03:51:31 rmind Exp $	*/
 /*-
  * Copyright (c) 2009-2014 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This material is based upon work partially supported by The
  * NetBSD Foundation under a contract with Mindaugas Rasiukevicius.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ *
  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 /*
  * Private NPF structures and interfaces.
  * For internal use within NPF core only.
  */
 #ifndef _NPF_IMPL_H_
 #define _NPF_IMPL_H_
 #if !defined(_KERNEL)
 #error "Kernel-level header only"
 #endif
 #ifdef _KERNEL_OPT
 /* For INET/INET6 definitions. */
 #include "opt_inet.h"
 #include "opt_inet6.h"
 #endif
 #include <sys/types.h>
 #include <sys/queue.h>
 #include <sys/hash.h>
 #include <sys/rbtree.h>
 #include <sys/ptree.h>
 #include <sys/rwlock.h>
 #include <net/bpf.h>
 #include <net/bpfjit.h>
 #include <net/if.h>
 #include "npf.h"
 #ifdef _NPF_DEBUG
 #define	NPF_PRINTF(x)	printf x
 #else
 #define	NPF_PRINTF(x)
 #endif
 /*
  * STRUCTURE DECLARATIONS.
  */
 struct npf_ruleset;
 struct npf_rule;
 struct npf_rprocset;
 struct npf_nat;
 struct npf_session;
 typedef struct npf_ruleset	npf_ruleset_t;
 typedef struct npf_rule		npf_rule_t;
 typedef struct npf_nat		npf_nat_t;
 typedef struct npf_rprocset	npf_rprocset_t;
 typedef struct npf_alg		npf_alg_t;
 typedef struct npf_natpolicy	npf_natpolicy_t;
 typedef struct npf_session	npf_session_t;
 struct npf_sehash;
 struct npf_table;
 struct npf_tableset;
 typedef struct npf_sehash	npf_sehash_t;
 typedef struct npf_table	npf_table_t;
 typedef struct npf_tableset	npf_tableset_t;
 /*
  * DEFINITIONS.
  */
 typedef void (*npf_workfunc_t)(void);
 /*
  * Some artificial limits.
  * Note: very unlikely to have many ALGs.
  */
 #define	NPF_MAX_RULES		(1024 * 1024)
 #define	NPF_MAX_ALGS		4
 #define	NPF_MAX_TABLES		128
 #define	NPF_MAX_RPROCS		128
 #define	NPF_MAX_IFMAP		64
 /*
  * SESSION STATE STRUCTURES
  */
 #define	NPF_FLOW_FORW		0
 #define	NPF_FLOW_BACK		1
 typedef struct {
 	uint32_t	nst_end;
 	uint32_t	nst_maxend;
 	uint32_t	nst_maxwin;
 	int		nst_wscale;
 } npf_tcpstate_t;
 typedef struct {
 	kmutex_t	nst_lock;
 	u_int		nst_state;
 	npf_tcpstate_t	nst_tcpst[2];
 } npf_state_t;
 /*
  * ALG FUNCTIONS.
  */
 typedef struct {
 	bool		(*match)(npf_cache_t *, nbuf_t *, npf_nat_t *, int);
 	bool		(*translate)(npf_cache_t *, nbuf_t *, npf_nat_t *, bool);
 	npf_session_t * (*inspect)(npf_cache_t *, nbuf_t *, int);
 } npfa_funcs_t;
 /*
  * INTERFACES.
  */
 /* NPF config, statistics, etc. */
 void		npf_config_init(void);
 void		npf_config_fini(void);
 void		npf_config_enter(void);
 void		npf_config_exit(void);
 void		npf_config_sync(void);
 bool		npf_config_locked_p(void);
 int		npf_config_read_enter(void);
 void		npf_config_read_exit(int);
 void		npf_config_reload(prop_dictionary_t, npf_ruleset_t *,
 		    npf_tableset_t *, npf_ruleset_t *, npf_rprocset_t *, bool);
 npf_ruleset_t *	npf_config_ruleset(void);
 npf_ruleset_t *	npf_config_natset(void);
 npf_tableset_t *npf_config_tableset(void);
 prop_dictionary_t npf_config_dict(void);
 bool		npf_default_pass(void);
 int		npf_worker_sysinit(void);
 void		npf_worker_sysfini(void);
 void		npf_worker_signal(void);
 void		npf_worker_register(npf_workfunc_t);
 void		npf_worker_unregister(npf_workfunc_t);
 void		npflogattach(int);
 void		npflogdetach(void);
 int		npfctl_switch(void *);
 int		npfctl_reload(u_long, void *);
 int		npfctl_getconf(u_long, void *);
 int		npfctl_sessions_save(u_long, void *);
 int		npfctl_sessions_load(u_long, void *);
 int		npfctl_rule(u_long, void *);
 int		npfctl_table(void *);
 void		npf_stats_inc(npf_stats_t);
 void		npf_stats_dec(npf_stats_t);
 u_int		npf_ifmap_register(const char *);
 void		npf_ifmap_flush(void);
 void		npf_ifmap_attach(ifnet_t *);
 void		npf_ifmap_detach(ifnet_t *);
 u_int		npf_ifmap_id(const ifnet_t *);
 /* Packet filter hooks. */
 int		npf_pfil_register(bool);
 void		npf_pfil_unregister(bool);
 bool		npf_pfil_registered_p(void);
 int		npf_packet_handler(void *, struct mbuf **, ifnet_t *, int);
 /* Protocol helpers. */
 int		npf_cache_all(npf_cache_t *, nbuf_t *);
 void		npf_recache(npf_cache_t *, nbuf_t *);
 bool		npf_rwrip(const npf_cache_t *, u_int, const npf_addr_t *);
 bool		npf_rwrport(const npf_cache_t *, u_int, const in_port_t);
 bool		npf_rwrcksum(const npf_cache_t *, u_int,
 		    const npf_addr_t *, const in_port_t);
 int		npf_napt_rwr(const npf_cache_t *, u_int, const npf_addr_t *,
 		    const in_addr_t);
 int		npf_npt66_rwr(const npf_cache_t *, u_int, const npf_addr_t *,
 		    npf_netmask_t, uint16_t);
 uint16_t	npf_fixup16_cksum(uint16_t, uint16_t, uint16_t);
 uint16_t	npf_fixup32_cksum(uint16_t, uint32_t, uint32_t);
 uint16_t	npf_addr_cksum(uint16_t, int, const npf_addr_t *,
 		    const npf_addr_t *);
 uint32_t	npf_addr_mix(const int, const npf_addr_t *, const npf_addr_t *);
 int		npf_addr_cmp(const npf_addr_t *, const npf_netmask_t,
 		    const npf_addr_t *, const npf_netmask_t, const int);
 void		npf_addr_mask(const npf_addr_t *, const npf_netmask_t,
 		    const int, npf_addr_t *);
 int		npf_tcpsaw(const npf_cache_t *, tcp_seq *, tcp_seq *,
 		    uint32_t *);
 bool		npf_fetch_tcpopts(npf_cache_t *, nbuf_t *, uint16_t *, int *);
 bool		npf_return_block(npf_cache_t *, nbuf_t *, const int);
 /* BPF interface. */
 void		npf_bpf_sysinit(void);
 void		npf_bpf_sysfini(void);
 int		npf_bpf_filter(bpf_args_t *, const void *, bpfjit_func_t);
 void *		npf_bpf_compile(void *, size_t);
 bool		npf_bpf_validate(const void *, size_t);
 /* Tableset interface. */
 void		npf_tableset_sysinit(void);
 void		npf_tableset_sysfini(void);
 extern const pt_tree_ops_t npf_table_ptree_ops;
 npf_tableset_t *npf_tableset_create(u_int);
 void		npf_tableset_destroy(npf_tableset_t *);
 int		npf_tableset_insert(npf_tableset_t *, npf_table_t *);
 npf_table_t *	npf_tableset_getbyname(npf_tableset_t *, const char *);
 npf_table_t *	npf_tableset_getbyid(npf_tableset_t *, u_int);
 void		npf_tableset_reload(npf_tableset_t *, npf_tableset_t *);
 void		npf_tableset_syncdict(const npf_tableset_t *, prop_dictionary_t);
 npf_table_t *	npf_table_create(const char *, u_int, int, void *, size_t);
 void		npf_table_destroy(npf_table_t *);
 int		npf_table_check(npf_tableset_t *, const char *, u_int, int);
 int		npf_table_insert(npf_table_t *, const int,
 		    const npf_addr_t *, const npf_netmask_t);
 int		npf_table_remove(npf_table_t *, const int,
 		    const npf_addr_t *, const npf_netmask_t);
 int		npf_table_lookup(npf_table_t *, const int, const npf_addr_t *);
 int		npf_table_list(npf_table_t *, void *, size_t);
 int		npf_table_flush(npf_table_t *);
 /* Ruleset interface. */
 npf_ruleset_t *	npf_ruleset_create(size_t);
 void		npf_ruleset_destroy(npf_ruleset_t *);
 void		npf_ruleset_insert(npf_ruleset_t *, npf_rule_t *);
 void		npf_ruleset_reload(npf_ruleset_t *, npf_ruleset_t *);
 void		npf_ruleset_natreload(npf_ruleset_t *, npf_ruleset_t *);
 npf_rule_t *	npf_ruleset_matchnat(npf_ruleset_t *, npf_natpolicy_t *);
 npf_rule_t *	npf_ruleset_sharepm(npf_ruleset_t *, npf_natpolicy_t *);
 void		npf_ruleset_freealg(npf_ruleset_t *, npf_alg_t *);
 int		npf_ruleset_add(npf_ruleset_t *, const char *, npf_rule_t *);
 int		npf_ruleset_remove(npf_ruleset_t *, const char *, uint64_t);
 int		npf_ruleset_remkey(npf_ruleset_t *, const char *,
 		    const void *, size_t);
 prop_dictionary_t npf_ruleset_list(npf_ruleset_t *, const char *);
 int		npf_ruleset_flush(npf_ruleset_t *, const char *);
 void		npf_ruleset_gc(npf_ruleset_t *);
 npf_rule_t *	npf_ruleset_inspect(npf_cache_t *, nbuf_t *,
 		    const npf_ruleset_t *, const int, const int);
 int		npf_rule_conclude(const npf_rule_t *, int *);
 /* Rule interface. */
 npf_rule_t *	npf_rule_alloc(prop_dictionary_t);
 void		npf_rule_setcode(npf_rule_t *, int, void *, size_t);
 void		npf_rule_setrproc(npf_rule_t *, npf_rproc_t *);
 void		npf_rule_free(npf_rule_t *);
 uint64_t	npf_rule_getid(const npf_rule_t *);
 npf_natpolicy_t *npf_rule_getnat(const npf_rule_t *);
 void		npf_rule_setnat(npf_rule_t *, npf_natpolicy_t *);
 npf_rproc_t *	npf_rule_getrproc(const npf_rule_t *);
 void		npf_ext_sysinit(void);
 void		npf_ext_sysfini(void);
 int		npf_ext_construct(const char *,
 		    npf_rproc_t *, prop_dictionary_t);
 npf_rprocset_t *npf_rprocset_create(void);
 void		npf_rprocset_destroy(npf_rprocset_t *);
 npf_rproc_t *	npf_rprocset_lookup(npf_rprocset_t *, const char *);
 void		npf_rprocset_insert(npf_rprocset_t *, npf_rproc_t *);
 npf_rproc_t *	npf_rproc_create(prop_dictionary_t);
 void		npf_rproc_acquire(npf_rproc_t *);
 void		npf_rproc_release(npf_rproc_t *);
 void		npf_rproc_run(npf_cache_t *, nbuf_t *, npf_rproc_t *, int *);
 /* Session handling interface. */
 void		npf_session_sysinit(void);
 void		npf_session_sysfini(void);
 void		npf_session_tracking(bool);
 npf_sehash_t *	sess_htable_create(void);
 void		sess_htable_destroy(npf_sehash_t *);
 npf_session_t *	npf_session_lookup(const npf_cache_t *, const nbuf_t *,
 		    const int, bool *);
 npf_session_t *	npf_session_inspect(npf_cache_t *, nbuf_t *, const int, int *);
 npf_session_t *	npf_session_establish(npf_cache_t *, nbuf_t *, const int);
 void		npf_session_release(npf_session_t *);
 void		npf_session_expire(npf_session_t *);
 bool		npf_session_pass(const npf_session_t *, npf_rproc_t **);
 void		npf_session_setpass(npf_session_t *, npf_rproc_t *);
 int		npf_session_setnat(npf_session_t *, npf_nat_t *, u_int);
 npf_nat_t *	npf_session_retnat(npf_session_t *, const int, bool *);
 void		npf_session_load(npf_sehash_t *);
 int		npf_session_save(prop_array_t, prop_array_t);
 int		npf_session_restore(npf_sehash_t *, prop_dictionary_t);
 /* State handling. */
 bool		npf_state_init(npf_cache_t *, nbuf_t *, npf_state_t *);
 bool		npf_state_inspect(npf_cache_t *, nbuf_t *, npf_state_t *,
 		    const bool);
 int		npf_state_etime(const npf_state_t *, const int);
 void		npf_state_destroy(npf_state_t *);
 bool		npf_state_tcp(npf_cache_t *, nbuf_t *, npf_state_t *, int);
 int		npf_state_tcp_timeout(const npf_state_t *);
 /* NAT. */
 void		npf_nat_sysinit(void);
 void		npf_nat_sysfini(void);
 npf_natpolicy_t *npf_nat_newpolicy(prop_dictionary_t, npf_ruleset_t *);
 void		npf_nat_freepolicy(npf_natpolicy_t *);
 bool		npf_nat_matchpolicy(npf_natpolicy_t *, npf_natpolicy_t *);
 bool		npf_nat_sharepm(npf_natpolicy_t *, npf_natpolicy_t *);
 void		npf_nat_freealg(npf_natpolicy_t *, npf_alg_t *);
 int		npf_do_nat(npf_cache_t *, npf_session_t *, nbuf_t *, const int);
 int		npf_nat_translate(npf_cache_t *, nbuf_t *, npf_nat_t *, bool);
 void		npf_nat_destroy(npf_nat_t *);
 void		npf_nat_getorig(npf_nat_t *, npf_addr_t **, in_port_t *);
 void		npf_nat_gettrans(npf_nat_t *, npf_addr_t **, in_port_t *);
 void		npf_nat_setalg(npf_nat_t *, npf_alg_t *, uintptr_t);
 int		npf_nat_save(prop_dictionary_t, prop_array_t, npf_nat_t *);
 npf_nat_t *	npf_nat_restore(prop_dictionary_t, npf_session_t *);
 /* ALG interface. */
 void		npf_alg_sysinit(void);
 void		npf_alg_sysfini(void);
 npf_alg_t *	npf_alg_register(const char *, const npfa_funcs_t *);
 int		npf_alg_unregister(npf_alg_t *);
 npf_alg_t *	npf_alg_construct(const char *);
 bool		npf_alg_match(npf_cache_t *, nbuf_t *, npf_nat_t *, int);
 void		npf_alg_exec(npf_cache_t *, nbuf_t *, npf_nat_t *, bool);
 npf_session_t *	npf_alg_session(npf_cache_t *, nbuf_t *, int);
 /* Debugging routines. */
 void		npf_addr_dump(const npf_addr_t *);
 void		npf_sessions_dump(void);
 void		npf_state_dump(const npf_state_t *);
 void		npf_nat_dump(const npf_nat_t *);
 void		npf_state_setsampler(void (*)(npf_state_t *, bool));
 #endif	/* _NPF_IMPL_H_ */

 @@ -1,702 +1,746 @@
-/*	$NetBSD: npf_inet.c,v 1.29 2014/02/13 03:34:40 rmind Exp $	*/
+/*	$NetBSD: npf_inet.c,v 1.30 2014/02/19 03:51:31 rmind Exp $	*/
 /*-
  * Copyright (c) 2009-2014 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This material is based upon work partially supported by The
  * NetBSD Foundation under a contract with Mindaugas Rasiukevicius.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ *
  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 /*
  * Various protocol related helper routines.
+ *
  * This layer manipulates npf_cache_t structure i.e. caches requested headers
  * and stores which information was cached in the information bit field.
  * It is also responsibility of this layer to update or invalidate the cache
  * on rewrites (e.g. by translation routines).
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: npf_inet.c,v 1.29 2014/02/13 03:34:40 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: npf_inet.c,v 1.30 2014/02/19 03:51:31 rmind Exp $");
 #include <sys/param.h>
 #include <sys/types.h>
 #include <net/pfil.h>
 #include <net/if.h>
 #include <net/ethertypes.h>
 #include <net/if_ether.h>
 #include <netinet/in_systm.h>
 #include <netinet/in.h>
 #include <netinet/ip.h>
 #include <netinet/ip6.h>
 #include <netinet/tcp.h>
 #include <netinet/udp.h>
 #include <netinet/ip_icmp.h>
 #include "npf_impl.h"
 /*
  * npf_fixup{16,32}_cksum: incremental update of the Internet checksum.
  */
 uint16_t
 npf_fixup16_cksum(uint16_t cksum, uint16_t odatum, uint16_t ndatum)
+{
 	uint32_t sum;
 	/*
 	 * RFC 1624:
 	 *	HC' = ~(~HC + ~m + m')
+	 *
 	 * Note: 1's complement sum is endian-independent (RFC 1071, page 2).
 	 */
 	sum = ~cksum & 0xffff;
 	sum += (~odatum & 0xffff) + ndatum;
 	sum = (sum >> 16) + (sum & 0xffff);
 	sum += (sum >> 16);
 	return ~sum & 0xffff;
+}
 uint16_t
 npf_fixup32_cksum(uint16_t cksum, uint32_t odatum, uint32_t ndatum)
+{
 	uint32_t sum;
 	/*
 	 * Checksum 32-bit datum as as two 16-bit.  Note, the first
 	 * 32->16 bit reduction is not necessary.
 	 */
 	sum = ~cksum & 0xffff;
 	sum += (~odatum & 0xffff) + (ndatum & 0xffff);
 	sum += (~odatum >> 16) + (ndatum >> 16);
 	sum = (sum >> 16) + (sum & 0xffff);
 	sum += (sum >> 16);
 	return ~sum & 0xffff;
+}
 /*
  * npf_addr_cksum: calculate checksum of the address, either IPv4 or IPv6.
  */
 uint16_t
 npf_addr_cksum(uint16_t cksum, int sz, const npf_addr_t *oaddr,
     const npf_addr_t *naddr)
+{
 	const uint32_t *oip32 = (const uint32_t *)oaddr;
 	const uint32_t *nip32 = (const uint32_t *)naddr;
 	KASSERT(sz % sizeof(uint32_t) == 0);
 	do {
 		cksum = npf_fixup32_cksum(cksum, *oip32++, *nip32++);
 		sz -= sizeof(uint32_t);
 	} while (sz);
 	return cksum;
+}
 /*
  * npf_addr_sum: provide IP addresses as a XORed 32-bit integer.
  * Note: used for hash function.
  */
 uint32_t
 npf_addr_mix(const int sz, const npf_addr_t *a1, const npf_addr_t *a2)
+{
 	uint32_t mix = 0;
 	KASSERT(sz > 0 && a1 != NULL && a2 != NULL);
 	for (int i = 0; i < (sz >> 2); i++) {
 		mix ^= a1->s6_addr32[i];
 		mix ^= a2->s6_addr32[i];
+	}
 	return mix;
+}
 /*
  * npf_addr_mask: apply the mask to a given address and store the result.
  */
 void
 npf_addr_mask(const npf_addr_t *addr, const npf_netmask_t mask,
     const int alen, npf_addr_t *out)
+{
 	const int nwords = alen >> 2;
 	uint_fast8_t length = mask;
 	/* Note: maximum length is 32 for IPv4 and 128 for IPv6. */
 	KASSERT(length <= NPF_MAX_NETMASK);
 	for (int i = 0; i < nwords; i++) {
 		uint32_t wordmask;
 		if (length >= 32) {
 			wordmask = htonl(0xffffffff);
 			length -= 32;
 		} else if (length) {
 			wordmask = htonl(0xffffffff << (32 - length));
 			length = 0;
 		} else {
 			wordmask = 0;
+		}
 		out->s6_addr32[i] = addr->s6_addr32[i] & wordmask;
+	}
+}
 /*
  * npf_addr_cmp: compare two addresses, either IPv4 or IPv6.
+ *
  * => Return 0 if equal and negative/positive if less/greater accordingly.
  * => Ignore the mask, if NPF_NO_NETMASK is specified.
  */
 int
 npf_addr_cmp(const npf_addr_t *addr1, const npf_netmask_t mask1,
     const npf_addr_t *addr2, const npf_netmask_t mask2, const int alen)
+{
 	npf_addr_t realaddr1, realaddr2;
 	if (mask1 != NPF_NO_NETMASK) {
 		npf_addr_mask(addr1, mask1, alen, &realaddr1);
 		addr1 = &realaddr1;
+	}
 	if (mask2 != NPF_NO_NETMASK) {
 		npf_addr_mask(addr2, mask2, alen, &realaddr2);
 		addr2 = &realaddr2;
+	}
 	return memcmp(addr1, addr2, alen);
+}
 /*
  * npf_tcpsaw: helper to fetch SEQ, ACK, WIN and return TCP data length.
+ *
  * => Returns all values in host byte-order.
  */
 int
 npf_tcpsaw(const npf_cache_t *npc, tcp_seq *seq, tcp_seq *ack, uint32_t *win)
+{
 	const struct tcphdr *th = npc->npc_l4.tcp;
 	u_int thlen;
 	KASSERT(npf_iscached(npc, NPC_TCP));
 	*seq = ntohl(th->th_seq);
 	*ack = ntohl(th->th_ack);
 	*win = (uint32_t)ntohs(th->th_win);
 	thlen = th->th_off << 2;
 	if (npf_iscached(npc, NPC_IP4)) {
 		const struct ip *ip = npc->npc_ip.v4;
 		return ntohs(ip->ip_len) - npc->npc_hlen - thlen;
 	} else if (npf_iscached(npc, NPC_IP6)) {
 		const struct ip6_hdr *ip6 = npc->npc_ip.v6;
 		return ntohs(ip6->ip6_plen) - thlen;
+	}
 	return 0;
+}
 /*
  * npf_fetch_tcpopts: parse and return TCP options.
  */
 bool
 npf_fetch_tcpopts(npf_cache_t *npc, nbuf_t *nbuf, uint16_t *mss, int *wscale)
+{
 	const struct tcphdr *th = npc->npc_l4.tcp;
 	int topts_len, step;
 	void *nptr;
 	uint8_t val;
 	bool ok;
 	KASSERT(npf_iscached(npc, NPC_IP46));
 	KASSERT(npf_iscached(npc, NPC_TCP));
 	/* Determine if there are any TCP options, get their length. */
 	topts_len = (th->th_off << 2) - sizeof(struct tcphdr);
 	if (topts_len <= 0) {
 		/* No options. */
 		return false;
+	}
 	KASSERT(topts_len <= MAX_TCPOPTLEN);
 	/* First step: IP and TCP header up to options. */
 	step = npc->npc_hlen + sizeof(struct tcphdr);
 	nbuf_reset(nbuf);
 next:
 	if ((nptr = nbuf_advance(nbuf, step, 1)) == NULL) {
 		ok = false;
 		goto done;
+	}
 	val = *(uint8_t *)nptr;
 	switch (val) {
 	case TCPOPT_EOL:
 		/* Done. */
 		ok = true;
 		goto done;
 	case TCPOPT_NOP:
 		topts_len--;
 		step = 1;
 		break;
 	case TCPOPT_MAXSEG:
 		if ((nptr = nbuf_advance(nbuf, 2, 2)) == NULL) {
 			ok = false;
 			goto done;
+		}
 		if (mss) {
 			if (*mss) {
 				memcpy(nptr, mss, sizeof(uint16_t));
 			} else {
 				memcpy(mss, nptr, sizeof(uint16_t));
+			}
+		}
 		topts_len -= TCPOLEN_MAXSEG;
 		step = 2;
 		break;
 	case TCPOPT_WINDOW:
 		/* TCP Window Scaling (RFC 1323). */
 		if ((nptr = nbuf_advance(nbuf, 2, 1)) == NULL) {
 			ok = false;
 			goto done;
+		}
 		val = *(uint8_t *)nptr;
 		*wscale = (val > TCP_MAX_WINSHIFT) ? TCP_MAX_WINSHIFT : val;
 		topts_len -= TCPOLEN_WINDOW;
 		step = 1;
 		break;
 	default:
 		if ((nptr = nbuf_advance(nbuf, 1, 1)) == NULL) {
 			ok = false;
 			goto done;
+		}
 		val = *(uint8_t *)nptr;
 		if (val < 2 || val > topts_len) {
 			ok = false;
 			goto done;
+		}
 		topts_len -= val;
 		step = val - 1;
+	}
 	/* Any options left? */
 	if (__predict_true(topts_len > 0)) {
 		goto next;
+	}
 	ok = true;
 done:
 	if (nbuf_flag_p(nbuf, NBUF_DATAREF_RESET)) {
 		npf_recache(npc, nbuf);
+	}
 	return ok;
+}
 static int
 npf_cache_ip(npf_cache_t *npc, nbuf_t *nbuf)
+{
 	const void *nptr = nbuf_dataptr(nbuf);
 	const uint8_t ver = *(const uint8_t *)nptr;
 	int flags = 0;
 	switch (ver >> 4) {
 	case IPVERSION: {
 		struct ip *ip;
 		ip = nbuf_ensure_contig(nbuf, sizeof(struct ip));
 		if (ip == NULL) {
 			return 0;
+		}
 		/* Check header length and fragment offset. */
 		if ((u_int)(ip->ip_hl << 2) < sizeof(struct ip)) {
 			return 0;
+		}
 		if (ip->ip_off & ~htons(IP_DF | IP_RF)) {
 			/* Note fragmentation. */
 			flags |= NPC_IPFRAG;
+		}
 		/* Cache: layer 3 - IPv4. */
 		npc->npc_alen = sizeof(struct in_addr);
 		npc->npc_ips[NPF_SRC] = (npf_addr_t *)&ip->ip_src;
 		npc->npc_ips[NPF_DST] = (npf_addr_t *)&ip->ip_dst;
 		npc->npc_hlen = ip->ip_hl << 2;
 		npc->npc_proto = ip->ip_p;
 		npc->npc_ip.v4 = ip;
 		flags |= NPC_IP4;
 		break;
+	}
 	case (IPV6_VERSION >> 4): {
 		struct ip6_hdr *ip6;
 		struct ip6_ext *ip6e;
 		size_t off, hlen;
 		ip6 = nbuf_ensure_contig(nbuf, sizeof(struct ip6_hdr));
 		if (ip6 == NULL) {
 			return 0;
+		}
 		/* Set initial next-protocol value. */
 		hlen = sizeof(struct ip6_hdr);
 		npc->npc_proto = ip6->ip6_nxt;
 		npc->npc_hlen = hlen;
 		/*
 		 * Advance by the length of the current header.
 		 */
 		off = nbuf_offset(nbuf);
 		while (nbuf_advance(nbuf, hlen, 0) != NULL) {
 			ip6e = nbuf_ensure_contig(nbuf, sizeof(*ip6e));
 			if (ip6e == NULL) {
 				return 0;
+			}
 			/*
 			 * Determine whether we are going to continue.
 			 */
 			switch (npc->npc_proto) {
 			case IPPROTO_HOPOPTS:
 			case IPPROTO_DSTOPTS:
 			case IPPROTO_ROUTING:
 				hlen = (ip6e->ip6e_len + 1) << 3;
 				break;
 			case IPPROTO_FRAGMENT:
 				hlen = sizeof(struct ip6_frag);
 				flags |= NPC_IPFRAG;
 				break;
 			case IPPROTO_AH:
 				hlen = (ip6e->ip6e_len + 2) << 2;
 				break;
 			default:
 				hlen = 0;
 				break;
+			}
 			if (!hlen) {
 				break;
+			}
 			npc->npc_proto = ip6e->ip6e_nxt;
 			npc->npc_hlen += hlen;
+		}
 		/*
 		 * Re-fetch the header pointers (nbufs might have been
 		 * reallocated).  Restore the original offset (if any).
 		 */
 		nbuf_reset(nbuf);
 		ip6 = nbuf_dataptr(nbuf);
 		if (off) {
 			nbuf_advance(nbuf, off, 0);
+		}
 		/* Cache: layer 3 - IPv6. */
 		npc->npc_alen = sizeof(struct in6_addr);
 		npc->npc_ips[NPF_SRC] = (npf_addr_t *)&ip6->ip6_src;
 		npc->npc_ips[NPF_DST]= (npf_addr_t *)&ip6->ip6_dst;
 		npc->npc_ip.v6 = ip6;
 		flags |= NPC_IP6;
 		break;
+	}
 	default:
 		break;
+	}
 	return flags;
+}
 /*
  * npf_cache_all: general routine to cache all relevant IP (v4 or v6)
  * and TCP, UDP or ICMP headers.
+ *
  * => nbuf offset shall be set accordingly.
  */
 int
 npf_cache_all(npf_cache_t *npc, nbuf_t *nbuf)
+{
 	int flags, l4flags;
 	u_int hlen;
 	/*
 	 * This routine is a main point where the references are cached,
 	 * therefore clear the flag as we reset.
 	 */
 again:
 	nbuf_unset_flag(nbuf, NBUF_DATAREF_RESET);
 	/*
 	 * First, cache the L3 header (IPv4 or IPv6).  If IP packet is
 	 * fragmented, then we cannot look into L4.
 	 */
 	flags = npf_cache_ip(npc, nbuf);
 	if ((flags & NPC_IP46) == 0 || (flags & NPC_IPFRAG) != 0) {
 		nbuf_unset_flag(nbuf, NBUF_DATAREF_RESET);
 		npc->npc_info |= flags;
 		return flags;
+	}
 	hlen = npc->npc_hlen;
 	switch (npc->npc_proto) {
 	case IPPROTO_TCP:
 		/* Cache: layer 4 - TCP. */
 		npc->npc_l4.tcp = nbuf_advance(nbuf, hlen,
 		    sizeof(struct tcphdr));
 		l4flags = NPC_LAYER4 | NPC_TCP;
 		break;
 	case IPPROTO_UDP:
 		/* Cache: layer 4 - UDP. */
 		npc->npc_l4.udp = nbuf_advance(nbuf, hlen,
 		    sizeof(struct udphdr));
 		l4flags = NPC_LAYER4 | NPC_UDP;
 		break;
 	case IPPROTO_ICMP:
 		/* Cache: layer 4 - ICMPv4. */
 		npc->npc_l4.icmp = nbuf_advance(nbuf, hlen,
 		    offsetof(struct icmp, icmp_void));
 		l4flags = NPC_LAYER4 | NPC_ICMP;
 		break;
 	case IPPROTO_ICMPV6:
 		/* Cache: layer 4 - ICMPv6. */
 		npc->npc_l4.icmp6 = nbuf_advance(nbuf, hlen,
 		    offsetof(struct icmp6_hdr, icmp6_data32));
 		l4flags = NPC_LAYER4 | NPC_ICMP;
 		break;
 	default:
 		l4flags = 0;
 		break;
+	}
 	if (nbuf_flag_p(nbuf, NBUF_DATAREF_RESET)) {
 		goto again;
+	}
 	/* Add the L4 flags if nbuf_advance() succeeded. */
 	if (l4flags && npc->npc_l4.hdr) {
 		flags |= l4flags;
+	}
 	npc->npc_info |= flags;
 	return flags;
+}
 void
 npf_recache(npf_cache_t *npc, nbuf_t *nbuf)
+{
 	const int mflags __diagused = npc->npc_info & (NPC_IP46 | NPC_LAYER4);
 	int flags __diagused;
 	nbuf_reset(nbuf);
 	npc->npc_info = 0;
 	flags = npf_cache_all(npc, nbuf);
 	KASSERT((flags & mflags) == mflags);
 	KASSERT(nbuf_flag_p(nbuf, NBUF_DATAREF_RESET) == 0);
+}
 /*
  * npf_rwrip: rewrite required IP address.
  */
 bool
 npf_rwrip(const npf_cache_t *npc, u_int which, const npf_addr_t *addr)
+{
 	KASSERT(npf_iscached(npc, NPC_IP46));
 	KASSERT(which == NPF_SRC || which == NPF_DST);
 	memcpy(npc->npc_ips[which], addr, npc->npc_alen);
 	return true;
+}
 /*
  * npf_rwrport: rewrite required TCP/UDP port.
  */
 bool
 npf_rwrport(const npf_cache_t *npc, u_int which, const in_port_t port)
+{
 	const int proto = npc->npc_proto;
 	in_port_t *oport;
 	KASSERT(npf_iscached(npc, NPC_TCP) || npf_iscached(npc, NPC_UDP));
 	KASSERT(proto == IPPROTO_TCP || proto == IPPROTO_UDP);
 	KASSERT(which == NPF_SRC || which == NPF_DST);
 	/* Get the offset and store the port in it. */
 	if (proto == IPPROTO_TCP) {
 		struct tcphdr *th = npc->npc_l4.tcp;
 		oport = (which == NPF_SRC) ? &th->th_sport : &th->th_dport;
 	} else {
 		struct udphdr *uh = npc->npc_l4.udp;
 		oport = (which == NPF_SRC) ? &uh->uh_sport : &uh->uh_dport;
+	}
 	memcpy(oport, &port, sizeof(in_port_t));
 	return true;
+}
 /*
  * npf_rwrcksum: rewrite IPv4 and/or TCP/UDP checksum.
  */
 bool
 npf_rwrcksum(const npf_cache_t *npc, u_int which,
     const npf_addr_t *addr, const in_port_t port)
+{
 	const npf_addr_t *oaddr = npc->npc_ips[which];
 	const int proto = npc->npc_proto;
 	const int alen = npc->npc_alen;
 	uint16_t *ocksum;
 	in_port_t oport;
 	KASSERT(npf_iscached(npc, NPC_LAYER4));
 	KASSERT(which == NPF_SRC || which == NPF_DST);
 	if (npf_iscached(npc, NPC_IP4)) {
 		struct ip *ip = npc->npc_ip.v4;
 		uint16_t ipsum = ip->ip_sum;
 		/* Recalculate IPv4 checksum and rewrite. */
 		ip->ip_sum = npf_addr_cksum(ipsum, alen, oaddr, addr);
 	} else {
 		/* No checksum for IPv6. */
 		KASSERT(npf_iscached(npc, NPC_IP6));
+	}
 	/* Nothing else to do for ICMP. */
-	if (proto == IPPROTO_ICMP) {
+	if (proto == IPPROTO_ICMP || proto == IPPROTO_ICMPV6) {
 		return true;
+	}
 	KASSERT(npf_iscached(npc, NPC_TCP) || npf_iscached(npc, NPC_UDP));
 	/*
 	 * Calculate TCP/UDP checksum:
 	 * - Skip if UDP and the current checksum is zero.
 	 * - Fixup the IP address change.
 	 * - Fixup the port change, if required (non-zero).
 	 */
 	if (proto == IPPROTO_TCP) {
 		struct tcphdr *th = npc->npc_l4.tcp;
 		ocksum = &th->th_sum;
 		oport = (which == NPF_SRC) ? th->th_sport : th->th_dport;
 	} else {
 		struct udphdr *uh = npc->npc_l4.udp;
 		KASSERT(proto == IPPROTO_UDP);
 		ocksum = &uh->uh_sum;
 		if (*ocksum == 0) {
 			/* No need to update. */
 			return true;
+		}
 		oport = (which == NPF_SRC) ? uh->uh_sport : uh->uh_dport;
+	}
 	uint16_t cksum = npf_addr_cksum(*ocksum, alen, oaddr, addr);
 	if (port) {
 		cksum = npf_fixup16_cksum(cksum, oport, port);
+	}
 	/* Rewrite TCP/UDP checksum. */
 	memcpy(ocksum, &cksum, sizeof(uint16_t));
 	return true;
+}
 /*
  * npf_napt_rwr: perform address and/or port translation.
  */
 int
 npf_napt_rwr(const npf_cache_t *npc, u_int which,
     const npf_addr_t *addr, const in_addr_t port)
+{
 	const unsigned proto = npc->npc_proto;
 	/*
 	 * Rewrite IP and/or TCP/UDP checksums first, since we need the
 	 * current (old) address/port for the calculations.  Then perform
 	 * the address translation i.e. rewrite source or destination.
 	 */
 	if (!npf_rwrcksum(npc, which, addr, port)) {
 		return EINVAL;
+	}
 	if (!npf_rwrip(npc, which, addr)) {
 		return EINVAL;
+	}
 	if (port == 0) {
 		/* Done. */
 		return 0;
+	}
 	switch (proto) {
 	case IPPROTO_TCP:
 	case IPPROTO_UDP:
 		/* Rewrite source/destination port. */
 		if (!npf_rwrport(npc, which, port)) {
 			return EINVAL;
+		}
 		break;
 	case IPPROTO_ICMP:
 	case IPPROTO_ICMPV6:
 		KASSERT(npf_iscached(npc, NPC_ICMP));
 		/* Nothing. */
 		break;
 	default:
 		return ENOTSUP;
+	}
 	return 0;
+}
 /*
  * IPv6-to-IPv6 Network Prefix Translation (NPTv6), as per RFC 6296.
  */
 int
 npf_npt66_rwr(const npf_cache_t *npc, u_int which, const npf_addr_t *pref,
     npf_netmask_t len, uint16_t adj)
+{
 	npf_addr_t *addr = npc->npc_ips[which];
 	unsigned remnant, word, preflen = len >> 4;
 	uint32_t sum;
 	KASSERT(which == NPF_SRC || which == NPF_DST);
 	if (!npf_iscached(npc, NPC_IP6)) {
 		return EINVAL;
+	}
 	if (len <= 48) {
 		/*
 		 * The word to adjust.  Cannot translate the 0xffff
 		 * subnet if /48 or shorter.
 		 */
 		word = 3;
 		if (addr->s6_addr16[word] == 0xffff) {
 			return EINVAL;
+		}
 	} else {
 		/*
 		 * Also, all 0s or 1s in the host part are disallowed for
 		 * longer than /48 prefixes.
 		 */
 		if ((addr->s6_addr32[2] == 0 && addr->s6_addr32[3] == 0) ||
 		    (addr->s6_addr32[2] == ~0U && addr->s6_addr32[3] == ~0U))
 			return EINVAL;
 		/* Determine the 16-bit word to adjust. */
 		for (word = 4; word < 8; word++)
 			if (addr->s6_addr16[word] != 0xffff)
 				break;
+	}
 	/* Rewrite the prefix. */
 	for (unsigned i = 0; i < preflen; i++) {
 		addr->s6_addr16[i] = pref->s6_addr16[i];
+	}
 	/*
 	 * If prefix length is within a 16-bit word (not dividable by 16),
 	 * then prepare a mask, determine the word and adjust it.
 	 */
 	if ((remnant = len - (preflen << 4)) != 0) {
 		const uint16_t wordmask = (1U << remnant) - 1;
 		const unsigned i = preflen;
 		addr->s6_addr16[i] = (pref->s6_addr16[i] & wordmask) |
 		    (addr->s6_addr16[i] & ~wordmask);
+	}
 	/*
 	 * Performing 1's complement sum/difference.
 	 */
 	sum = addr->s6_addr16[word] + adj;
 	while (sum >> 16) {
 		sum = (sum >> 16) + (sum & 0xffff);
+	}
 	if (sum == 0xffff) {
 		/* RFC 1071. */
 		sum = 0x0000;
+	}
 	addr->s6_addr16[word] = sum;
 	return 0;
+}
 #if defined(DDB) || defined(_NPF_TESTING)
 void
 npf_addr_dump(const npf_addr_t *addr)
+{
 	printf("IP[%x:%x:%x:%x]\n",
 	    addr->s6_addr32[0], addr->s6_addr32[1],
 	    addr->s6_addr32[2], addr->s6_addr32[3]);
+}
 #endif

 @@ -1,946 +1,903 @@
-/*	$NetBSD: npf_nat.c,v 1.25 2014/02/13 03:34:40 rmind Exp $	*/
+/*	$NetBSD: npf_nat.c,v 1.26 2014/02/19 03:51:31 rmind Exp $	*/
 /*-
  * Copyright (c) 2014 Mindaugas Rasiukevicius <rmind at netbsd org>
  * Copyright (c) 2010-2013 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This material is based upon work partially supported by The
  * NetBSD Foundation under a contract with Mindaugas Rasiukevicius.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ *
  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 /*
  * NPF network address port translation (NAPT) and other forms of NAT.
  * Described in RFC 2663, RFC 3022, etc.
+ *
  * Overview
+ *
  *	There are few mechanisms: NAT policy, port map and translation.
  *	NAT module has a separate ruleset, where rules contain associated
  *	NAT policy, thus flexible filter criteria can be used.
+ *
  * Translation types
+ *
  *	There are two types of translation: outbound (NPF_NATOUT) and
  *	inbound (NPF_NATIN).  It should not be confused with connection
  *	direction.  See npf_nat_which() for the description of how the
  *	addresses are rewritten.
+ *
  *	It should be noted that bi-directional NAT is a combined outbound
  *	and inbound translation, therefore constructed as two policies.
+ *
  * NAT policies and port maps
+ *
  *	NAT (translation) policy is applied when a packet matches the rule.
  *	Apart from filter criteria, NAT policy has a translation IP address
  *	and associated port map.  Port map is a bitmap used to reserve and
  *	use unique TCP/UDP ports for translation.  Port maps are unique to
  *	the IP addresses, therefore multiple NAT policies with the same IP
  *	will share the same port map.
+ *
  * Sessions, translation entries and their life-cycle
+ *
  *	NAT module relies on session management module.  Each translated
  *	session has an associated translation entry (npf_nat_t), which
  *	contains information used for backwards stream translation, i.e.
  *	original IP address with port and translation port, allocated from
  *	the port map.  Each NAT entry is associated with the policy, which
  *	contains translation IP address.  Allocated port is returned to the
  *	port map and NAT entry is destroyed when session expires.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: npf_nat.c,v 1.25 2014/02/13 03:34:40 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: npf_nat.c,v 1.26 2014/02/19 03:51:31 rmind Exp $");
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/atomic.h>
 #include <sys/bitops.h>
 #include <sys/condvar.h>
 #include <sys/kmem.h>
 #include <sys/mutex.h>
 #include <sys/pool.h>
 #include <sys/proc.h>
 #include <sys/cprng.h>
 #include <net/pfil.h>
 #include <netinet/in.h>
 #include "npf_impl.h"
 /*
  * NPF portmap structure.
  */
 typedef struct {
 	u_int			p_refcnt;
 	uint32_t		p_bitmap[0];
 } npf_portmap_t;
 /* Portmap range: [ 1024 .. 65535 ] */
 #define	PORTMAP_FIRST		(1024)
 #define	PORTMAP_SIZE		((65536 - PORTMAP_FIRST) / 32)
 #define	PORTMAP_FILLED		((uint32_t)~0U)
 #define	PORTMAP_MASK		(31)
 #define	PORTMAP_SHIFT		(5)
 #define	PORTMAP_MEM_SIZE	\
     (sizeof(npf_portmap_t) + (PORTMAP_SIZE * sizeof(uint32_t)))
 /*
  * NAT policy structure.
  */
 struct npf_natpolicy {
 	LIST_HEAD(, npf_nat)	n_nat_list;
 	volatile u_int		n_refcnt;
 	kmutex_t		n_lock;
 	kcondvar_t		n_cv;
 	npf_portmap_t *		n_portmap;
 	/* NPF_NP_CMP_START */
 	int			n_type;
 	u_int			n_flags;
 	size_t			n_addr_sz;
 	npf_addr_t		n_taddr;
 	npf_netmask_t		n_tmask;
 	in_port_t		n_tport;
 	u_int			n_algo;
 	union {
 		uint16_t	n_npt66_adj;
 	};
 };
 #define	NPF_NP_CMP_START	offsetof(npf_natpolicy_t, n_type)
 #define	NPF_NP_CMP_SIZE		(sizeof(npf_natpolicy_t) - NPF_NP_CMP_START)
 /*
  * NAT translation entry for a session.
  */
 struct npf_nat {
 	/* Association (list entry and a link pointer) with NAT policy. */
 	LIST_ENTRY(npf_nat)	nt_entry;
 	npf_natpolicy_t *	nt_natpolicy;
 	npf_session_t *		nt_session;
 	/* Original address and port (for backwards translation). */
 	npf_addr_t		nt_oaddr;
 	in_port_t		nt_oport;
 	/* Translation port (for redirects). */
 	in_port_t		nt_tport;
 	/* ALG (if any) associated with this NAT entry. */
 	npf_alg_t *		nt_alg;
 	uintptr_t		nt_alg_arg;
 };
 static pool_cache_t		nat_cache	__read_mostly;
 /*
  * npf_nat_sys{init,fini}: initialise/destroy NAT subsystem structures.
  */
 void
 npf_nat_sysinit(void)
+{
 	nat_cache = pool_cache_init(sizeof(npf_nat_t), coherency_unit,
 , 0, "npfnatpl", NULL, IPL_NET, NULL, NULL, NULL);
 	KASSERT(nat_cache != NULL);
+}
 void
 npf_nat_sysfini(void)
+{
 	/* All NAT policies should already be destroyed. */
 	pool_cache_destroy(nat_cache);
+}
 /*
  * npf_nat_newpolicy: create a new NAT policy.
+ *
  * => Shares portmap if policy is on existing translation address.
  */
 npf_natpolicy_t *
 npf_nat_newpolicy(prop_dictionary_t natdict, npf_ruleset_t *nrlset)
+{
 	npf_natpolicy_t *np;
 	prop_object_t obj;
 	npf_portmap_t *pm;
 	np = kmem_zalloc(sizeof(npf_natpolicy_t), KM_SLEEP);
 	/* Translation type and flags. */
 	prop_dictionary_get_int32(natdict, "type", &np->n_type);
 	prop_dictionary_get_uint32(natdict, "flags", &np->n_flags);
 	/* Should be exclusively either inbound or outbound NAT. */
 	if (((np->n_type == NPF_NATIN) ^ (np->n_type == NPF_NATOUT)) == 0) {
 		goto err;
+	}
 	mutex_init(&np->n_lock, MUTEX_DEFAULT, IPL_SOFTNET);
 	cv_init(&np->n_cv, "npfnatcv");
 	LIST_INIT(&np->n_nat_list);
 	/* Translation IP, mask and port (if applicable). */
 	obj = prop_dictionary_get(natdict, "translation-ip");
 	np->n_addr_sz = prop_data_size(obj);
 	if (np->n_addr_sz == 0 || np->n_addr_sz > sizeof(npf_addr_t)) {
 		goto err;
+	}
 	memcpy(&np->n_taddr, prop_data_data_nocopy(obj), np->n_addr_sz);
 	prop_dictionary_get_uint8(natdict, "translation-mask", &np->n_tmask);
 	prop_dictionary_get_uint16(natdict, "translation-port", &np->n_tport);
 	prop_dictionary_get_uint32(natdict, "translation-algo", &np->n_algo);
 	switch (np->n_algo) {
 	case NPF_ALGO_NPT66:
 		prop_dictionary_get_uint16(natdict, "npt66-adjustment",
 		    &np->n_npt66_adj);
 		break;
 	default:
 		if (np->n_tmask != NPF_NO_NETMASK)
 			goto err;
 		break;
+	}
 	/* Determine if port map is needed. */
 	np->n_portmap = NULL;
 	if ((np->n_flags & NPF_NAT_PORTMAP) == 0) {
 		/* No port map. */
 		return np;
+	}
 	/*
 	 * Inspect NAT policies in the ruleset for port map sharing.
 	 * Note that npf_ruleset_sharepm() will increase the reference count.
 	 */
 	if (!npf_ruleset_sharepm(nrlset, np)) {
 		/* Allocate a new port map for the NAT policy. */
 		pm = kmem_zalloc(PORTMAP_MEM_SIZE, KM_SLEEP);
 		pm->p_refcnt = 1;
 		KASSERT((uintptr_t)pm->p_bitmap == (uintptr_t)pm + sizeof(*pm));
 		np->n_portmap = pm;
 	} else {
 		KASSERT(np->n_portmap != NULL);
+	}
 	return np;
 err:
 	kmem_free(np, sizeof(npf_natpolicy_t));
 	return NULL;
+}
 /*
  * npf_nat_freepolicy: free NAT policy and, on last reference, free portmap.
+ *
  * => Called from npf_rule_free() during the reload via npf_ruleset_destroy().
  */
 void
 npf_nat_freepolicy(npf_natpolicy_t *np)
+{
 	npf_portmap_t *pm = np->n_portmap;
 	npf_session_t *se;
 	npf_nat_t *nt;
 	/*
 	 * Disassociate all entries from the policy.  At this point,
 	 * new entries can no longer be created for this policy.
 	 */
 	mutex_enter(&np->n_lock);
 	LIST_FOREACH(nt, &np->n_nat_list, nt_entry) {
 		se = nt->nt_session;
 		KASSERT(se != NULL);
 		npf_session_expire(se);
+	}
 	while (!LIST_EMPTY(&np->n_nat_list)) {
 		cv_wait(&np->n_cv, &np->n_lock);
+	}
 	mutex_exit(&np->n_lock);
 	/* Kick the worker - all references should be going away. */
 	npf_worker_signal();
 	while (np->n_refcnt) {
 		kpause("npfgcnat", false, 1, NULL);
+	}
 	KASSERT(LIST_EMPTY(&np->n_nat_list));
 	/* Destroy the port map, on last reference. */
 	if (pm && --pm->p_refcnt == 0) {
 		KASSERT((np->n_flags & NPF_NAT_PORTMAP) != 0);
 		kmem_free(pm, PORTMAP_MEM_SIZE);
+	}
 	cv_destroy(&np->n_cv);
 	mutex_destroy(&np->n_lock);
 	kmem_free(np, sizeof(npf_natpolicy_t));
+}
 void
 npf_nat_freealg(npf_natpolicy_t *np, npf_alg_t *alg)
+{
 	npf_nat_t *nt;
 	mutex_enter(&np->n_lock);
 	LIST_FOREACH(nt, &np->n_nat_list, nt_entry) {
 		if (nt->nt_alg != alg) {
 			continue;
+		}
 		nt->nt_alg = NULL;
+	}
 	mutex_exit(&np->n_lock);
+}
 /*
  * npf_nat_matchpolicy: compare two NAT policies.
+ *
  * => Return 0 on match, and non-zero otherwise.
  */
 bool
 npf_nat_matchpolicy(npf_natpolicy_t *np, npf_natpolicy_t *mnp)
+{
 	void *np_raw, *mnp_raw;
 	/*
 	 * Compare the relevant NAT policy information (in raw form),
 	 * which is enough for matching criterion.
 	 */
 	KASSERT(np && mnp && np != mnp);
 	np_raw = (uint8_t *)np + NPF_NP_CMP_START;
 	mnp_raw = (uint8_t *)mnp + NPF_NP_CMP_START;
 	return (memcmp(np_raw, mnp_raw, NPF_NP_CMP_SIZE) == 0);
+}
 bool
 npf_nat_sharepm(npf_natpolicy_t *np, npf_natpolicy_t *mnp)
+{
 	npf_portmap_t *pm, *mpm;
 	KASSERT(np && mnp && np != mnp);
 	/* Using port map and having equal translation address? */
 	if ((np->n_flags & mnp->n_flags & NPF_NAT_PORTMAP) == 0) {
 		return false;
+	}
 	if (np->n_addr_sz != mnp->n_addr_sz) {
 		return false;
+	}
 	if (memcmp(&np->n_taddr, &mnp->n_taddr, np->n_addr_sz) != 0) {
 		return false;
+	}
 	/* If NAT policy has an old port map - drop the reference. */
 	mpm = mnp->n_portmap;
 	if (mpm) {
 		/* Note: at this point we cannot hold a last reference. */
 		KASSERT(mpm->p_refcnt > 1);
 		mpm->p_refcnt--;
+	}
 	/* Share the port map. */
 	pm = np->n_portmap;
 	mnp->n_portmap = pm;
 	pm->p_refcnt++;
 	return true;
+}
 /*
  * npf_nat_getport: allocate and return a port in the NAT policy portmap.
+ *
  * => Returns in network byte-order.
  * => Zero indicates failure.
  */
 static in_port_t
 npf_nat_getport(npf_natpolicy_t *np)
+{
 	npf_portmap_t *pm = np->n_portmap;
 	u_int n = PORTMAP_SIZE, idx, bit;
 	uint32_t map, nmap;
 	idx = cprng_fast32() % PORTMAP_SIZE;
 	for (;;) {
 		KASSERT(idx < PORTMAP_SIZE);
 		map = pm->p_bitmap[idx];
 		if (__predict_false(map == PORTMAP_FILLED)) {
 			if (n-- == 0) {
 				/* No space. */
 				return 0;
+			}
 			/* This bitmap is filled, next. */
 			idx = (idx ? idx : PORTMAP_SIZE) - 1;
 			continue;
+		}
 		bit = ffs32(~map) - 1;
 		nmap = map | (1 << bit);
 		if (atomic_cas_32(&pm->p_bitmap[idx], map, nmap) == map) {
 			/* Success. */
 			break;
+		}
+	}
 	return htons(PORTMAP_FIRST + (idx << PORTMAP_SHIFT) + bit);
+}
 /*
  * npf_nat_takeport: allocate specific port in the NAT policy portmap.
  */
 static bool
 npf_nat_takeport(npf_natpolicy_t *np, in_port_t port)
+{
 	npf_portmap_t *pm = np->n_portmap;
 	uint32_t map, nmap;
 	u_int idx, bit;
 	port = ntohs(port) - PORTMAP_FIRST;
 	idx = port >> PORTMAP_SHIFT;
 	bit = port & PORTMAP_MASK;
 	map = pm->p_bitmap[idx];
 	nmap = map | (1 << bit);
 	if (map == nmap) {
 		/* Already taken. */
 		return false;
+	}
 	return atomic_cas_32(&pm->p_bitmap[idx], map, nmap) == map;
+}
 /*
  * npf_nat_putport: return port as available in the NAT policy portmap.
+ *
  * => Port should be in network byte-order.
  */
 static void
 npf_nat_putport(npf_natpolicy_t *np, in_port_t port)
+{
 	npf_portmap_t *pm = np->n_portmap;
 	uint32_t map, nmap;
 	u_int idx, bit;
 	port = ntohs(port) - PORTMAP_FIRST;
 	idx = port >> PORTMAP_SHIFT;
 	bit = port & PORTMAP_MASK;
 	do {
 		map = pm->p_bitmap[idx];
 		KASSERT(map | (1 << bit));
 		nmap = map & ~(1 << bit);
 	} while (atomic_cas_32(&pm->p_bitmap[idx], map, nmap) != map);
+}
 /*
  * npf_nat_which: tell which address (source or destination) should be
  * rewritten given the combination of the NAT type and flow direction.
  */
 static inline u_int
 npf_nat_which(const int type, bool forw)
+{
 	/*
 	 * Outbound NAT rewrites:
 	 * - Source (NPF_SRC) on "forwards" stream.
 	 * - Destination (NPF_DST) on "backwards" stream.
 	 * Inbound NAT is other way round.
 	 */
 	if (type == NPF_NATOUT) {
 		forw = !forw;
 	} else {
 		KASSERT(type == NPF_NATIN);
+	}
 	CTASSERT(NPF_SRC == 0 && NPF_DST == 1);
 	KASSERT(forw == NPF_SRC || forw == NPF_DST);
 	return (u_int)forw;
+}
 /*
  * npf_nat_inspect: inspect packet against NAT ruleset and return a policy.
+ *
  * => Acquire a reference on the policy, if found.
  */
 static npf_natpolicy_t *
 npf_nat_inspect(npf_cache_t *npc, nbuf_t *nbuf, const int di)
+{
 	int slock = npf_config_read_enter();
 	npf_ruleset_t *rlset = npf_config_natset();
 	npf_natpolicy_t *np;
 	npf_rule_t *rl;
 	rl = npf_ruleset_inspect(npc, nbuf, rlset, di, NPF_LAYER_3);
 	if (rl == NULL) {
 		npf_config_read_exit(slock);
 		return NULL;
+	}
 	np = npf_rule_getnat(rl);
 	atomic_inc_uint(&np->n_refcnt);
 	npf_config_read_exit(slock);
 	return np;
+}
 /*
  * npf_nat_create: create a new NAT translation entry.
  */
 static npf_nat_t *
 npf_nat_create(npf_cache_t *npc, npf_natpolicy_t *np, npf_session_t *se)
+{
 	const int proto = npc->npc_proto;
 	npf_nat_t *nt;
 	KASSERT(npf_iscached(npc, NPC_IP46));
 	KASSERT(npf_iscached(npc, NPC_LAYER4));
 	/* Construct a new NAT entry and associate it with the session. */
 	nt = pool_cache_get(nat_cache, PR_NOWAIT);
 	if (nt == NULL){
 		return NULL;
+	}
 	npf_stats_inc(NPF_STAT_NAT_CREATE);
 	nt->nt_natpolicy = np;
 	nt->nt_session = se;
 	nt->nt_alg = NULL;
 	/* Save the original address which may be rewritten. */
 	if (np->n_type == NPF_NATOUT) {
 		/* Outbound NAT: source (think internal) address. */
 		memcpy(&nt->nt_oaddr, npc->npc_ips[NPF_SRC], npc->npc_alen);
 	} else {
 		/* Inbound NAT: destination (think external) address. */
 		KASSERT(np->n_type == NPF_NATIN);
 		memcpy(&nt->nt_oaddr, npc->npc_ips[NPF_DST], npc->npc_alen);
+	}
 	/*
 	 * Port translation, if required, and if it is TCP/UDP.
 	 */
 	if ((np->n_flags & NPF_NAT_PORTS) == 0 ||
 	    (proto != IPPROTO_TCP && proto != IPPROTO_UDP)) {
 		nt->nt_oport = 0;
 		nt->nt_tport = 0;
 		goto out;
+	}
 	/* Save the relevant TCP/UDP port. */
 	if (proto == IPPROTO_TCP) {
 		const struct tcphdr *th = npc->npc_l4.tcp;
 		nt->nt_oport = (np->n_type == NPF_NATOUT) ?
 		    th->th_sport : th->th_dport;
 	} else {
 		const struct udphdr *uh = npc->npc_l4.udp;
 		nt->nt_oport = (np->n_type == NPF_NATOUT) ?
 		    uh->uh_sport : uh->uh_dport;
+	}
 	/* Get a new port for translation. */
 	if ((np->n_flags & NPF_NAT_PORTMAP) != 0) {
 		nt->nt_tport = npf_nat_getport(np);
 	} else {
 		nt->nt_tport = np->n_tport;
+	}
 out:
 	mutex_enter(&np->n_lock);
 	LIST_INSERT_HEAD(&np->n_nat_list, nt, nt_entry);
 	mutex_exit(&np->n_lock);
 	return nt;
+}
 /*
  * npf_nat_rwr: perform address and/or port translation.
  */
 static int
 npf_nat_rwr(npf_cache_t *npc, const npf_natpolicy_t *np,
     const npf_addr_t *addr, const in_addr_t port, bool forw)
 	const unsigned proto = npc->npc_proto;
 	const u_int which = npf_nat_which(np->n_type, forw);
 	/*
 	 * Rewrite IP and/or TCP/UDP checksums first, since we need the
 	 * current (old) address/port for the calculations.  Then perform
 	 * the address translation i.e. rewrite source or destination.
 	 */
 	if (!npf_rwrcksum(npc, which, addr, port)) {
 		return EINVAL;
 	if (!npf_rwrip(npc, which, addr)) {
 		return EINVAL;
 	if ((np->n_flags & NPF_NAT_PORTS) == 0) {
 		/* Done. */
 		return 0;
 	switch (proto) {
 	case IPPROTO_TCP:
 	case IPPROTO_UDP:
 		/* Rewrite source/destination port. */
 		if (!npf_rwrport(npc, which, port)) {
 			return EINVAL;
 		break;
 	case IPPROTO_ICMP:
 		KASSERT(npf_iscached(npc, NPC_ICMP));
 		/* Nothing. */
 		break;
 	default:
 		return ENOTSUP;
 	return 0;
 /*
  * npf_nat_translate: perform translation given the state data.
  */
-int
+static inline int
 npf_nat_translate(npf_cache_t *npc, nbuf_t *nbuf, npf_nat_t *nt, bool forw)
+{
 	const npf_natpolicy_t *np = nt->nt_natpolicy;
 	const u_int which = npf_nat_which(np->n_type, forw);
 	const npf_addr_t *addr;
 	in_port_t port;
 	KASSERT(npf_iscached(npc, NPC_IP46));
 	KASSERT(npf_iscached(npc, NPC_LAYER4));
 	KASSERT(!nbuf_flag_p(nbuf, NBUF_DATAREF_RESET));
 	if (forw) {
 		/* "Forwards" stream: use translation address/port. */
 		addr = &np->n_taddr;
 		port = nt->nt_tport;
 	} else {
 		/* "Backwards" stream: use original address/port. */
 		addr = &nt->nt_oaddr;
 		port = nt->nt_oport;
+	}
 	KASSERT((np->n_flags & NPF_NAT_PORTS) != 0 || port == 0);
-	/* Execute ALG hook first. */
+	/* Execute ALG translation first. */
 	if ((npc->npc_info & NPC_ALG_EXEC) == 0) {
 		npc->npc_info |= NPC_ALG_EXEC;
 		npf_alg_exec(npc, nbuf, nt, forw);
 		npf_recache(npc, nbuf);
+	}
 	KASSERT(!nbuf_flag_p(nbuf, NBUF_DATAREF_RESET));
 	/* Finally, perform the translation. */
-	return npf_nat_rwr(npc, np, addr, port, forw);
+	return npf_napt_rwr(npc, which, addr, port);
+}
 /*
  * npf_nat_algo: perform the translation given the algorithm.
  */
 static inline int
 npf_nat_algo(npf_cache_t *npc, const npf_natpolicy_t *np, bool forw)
+{
-	u_int which;
+	const u_int which = npf_nat_which(np->n_type, forw);
 	int error;
 	switch (np->n_algo) {
 	case NPF_ALGO_NPT66:
 		which = npf_nat_which(np->n_type, forw);
 		error = npf_npt66_rwr(npc, which, &np->n_taddr,
 		    np->n_tmask, np->n_npt66_adj);
 		break;
 	default:
-		error = npf_nat_rwr(npc, np, &np->n_taddr, np->n_tport, forw);
+		error = npf_napt_rwr(npc, which, &np->n_taddr, np->n_tport);
 		break;
+	}
 	return error;
+}
 /*
  * npf_do_nat:
  *	- Inspect packet for a NAT policy, unless a session with a NAT
  *	  association already exists.  In such case, determine whether it
  *	  is a "forwards" or "backwards" stream.
  *	- Perform translation: rewrite source or destination fields,
  *	  depending on translation type and direction.
  *	- Associate a NAT policy with a session (may establish a new).
  */
 int
 npf_do_nat(npf_cache_t *npc, npf_session_t *se, nbuf_t *nbuf, const int di)
+{
 	npf_session_t *nse = NULL;
 	npf_natpolicy_t *np;
 	npf_nat_t *nt;
 	int error;
 	bool forw;
 	/* All relevant IPv4 data should be already cached. */
 	if (!npf_iscached(npc, NPC_IP46) || !npf_iscached(npc, NPC_LAYER4)) {
 		return 0;
+	}
 	KASSERT(!nbuf_flag_p(nbuf, NBUF_DATAREF_RESET));
 	/*
 	 * Return the NAT entry associated with the session, if any.
 	 * Determines whether the stream is "forwards" or "backwards".
 	 * Note: no need to lock, since reference on session is held.
 	 */
 	if (se && (nt = npf_session_retnat(se, di, &forw)) != NULL) {
 		np = nt->nt_natpolicy;
 		goto translate;
+	}
 	/*
 	 * Inspect the packet for a NAT policy, if there is no session.
 	 * Note: acquires a reference if found.
 	 */
 	np = npf_nat_inspect(npc, nbuf, di);
 	if (np == NULL) {
 		/* If packet does not match - done. */
 		return 0;
+	}
 	forw = true;
 	/* Static NAT - just perform the translation. */
 	if (np->n_flags & NPF_NAT_STATIC) {
 		if (nbuf_cksum_barrier(nbuf, di)) {
 			npf_recache(npc, nbuf);
+		}
 		error = npf_nat_algo(npc, np, forw);
 		atomic_dec_uint(&np->n_refcnt);
 		return error;
+	}
 	/*
 	 * If there is no local session (no "stateful" rule - unusual, but
 	 * possible configuration), establish one before translation.  Note
 	 * that it is not a "pass" session, therefore passing of "backwards"
 	 * stream depends on other, stateless filtering rules.
 	 */
 	if (se == NULL) {
 		nse = npf_session_establish(npc, nbuf, di);
 		if (nse == NULL) {
 			atomic_dec_uint(&np->n_refcnt);
 			return ENOMEM;
+		}
 		se = nse;
+	}
 	/*
 	 * Create a new NAT entry and associate with the session.
 	 * We will consume the reference on success (release on error).
 	 */
 	nt = npf_nat_create(npc, np, se);
 	if (nt == NULL) {
 		atomic_dec_uint(&np->n_refcnt);
 		error = ENOMEM;
 		goto out;
+	}
 	/* Associate the NAT translation entry with the session. */
 	error = npf_session_setnat(se, nt, np->n_type);
 	if (error) {
 		/* Will release the reference. */
 		npf_nat_destroy(nt);
 		goto out;
+	}
 	/* Determine whether any ALG matches. */
 	if (npf_alg_match(npc, nbuf, nt, di)) {
 		KASSERT(nt->nt_alg != NULL);
+	}
 translate:
 	/* May need to process the delayed checksums first (XXX: NetBSD). */
 	if (nbuf_cksum_barrier(nbuf, di)) {
 		npf_recache(npc, nbuf);
+	}
 	/* Perform the translation. */
 	error = npf_nat_translate(npc, nbuf, nt, forw);
 out:
 	if (__predict_false(nse)) {
 		if (error) {
 			/* It created for NAT - just expire. */
 			npf_session_expire(nse);
+		}
 		npf_session_release(nse);
+	}
 	return error;
+}
 /*
  * npf_nat_gettrans: return translation IP address and port.
  */
 void
 npf_nat_gettrans(npf_nat_t *nt, npf_addr_t **addr, in_port_t *port)
+{
 	npf_natpolicy_t *np = nt->nt_natpolicy;
 	*addr = &np->n_taddr;
 	*port = nt->nt_tport;
+}
 /*
  * npf_nat_getorig: return original IP address and port from translation entry.
  */
 void
 npf_nat_getorig(npf_nat_t *nt, npf_addr_t **addr, in_port_t *port)
+{
 	*addr = &nt->nt_oaddr;
 	*port = nt->nt_oport;
+}
 /*
  * npf_nat_setalg: associate an ALG with the NAT entry.
  */
 void
 npf_nat_setalg(npf_nat_t *nt, npf_alg_t *alg, uintptr_t arg)
+{
 	nt->nt_alg = alg;
 	nt->nt_alg_arg = arg;
+}
 /*
  * npf_nat_destroy: destroy NAT structure (performed on session expiration).
  */
 void
 npf_nat_destroy(npf_nat_t *nt)
+{
 	npf_natpolicy_t *np = nt->nt_natpolicy;
 	/* Return any taken port to the portmap. */
 	if ((np->n_flags & NPF_NAT_PORTMAP) != 0 && nt->nt_tport) {
 		npf_nat_putport(np, nt->nt_tport);
+	}
 	mutex_enter(&np->n_lock);
 	LIST_REMOVE(nt, nt_entry);
 	if (LIST_EMPTY(&np->n_nat_list)) {
 		/* Notify any waiters if empty. */
 		cv_broadcast(&np->n_cv);
+	}
 	atomic_dec_uint(&np->n_refcnt);
 	mutex_exit(&np->n_lock);
 	pool_cache_put(nat_cache, nt);
 	npf_stats_inc(NPF_STAT_NAT_DESTROY);
+}
 /*
  * npf_nat_save: construct NAT entry and reference to the NAT policy.
  */
 int
 npf_nat_save(prop_dictionary_t sedict, prop_array_t natlist, npf_nat_t *nt)
+{
 	npf_natpolicy_t *np = nt->nt_natpolicy;
 	prop_object_iterator_t it;
 	prop_dictionary_t npdict;
 	prop_data_t nd, npd;
 	uint64_t itnp;
 	/* Set NAT entry data. */
 	nd = prop_data_create_data(nt, sizeof(npf_nat_t));
 	prop_dictionary_set(sedict, "nat-data", nd);
 	prop_object_release(nd);
 	/* Find or create a NAT policy. */
 	it = prop_array_iterator(natlist);
 	while ((npdict = prop_object_iterator_next(it)) != NULL) {
 		CTASSERT(sizeof(uintptr_t) <= sizeof(uint64_t));
 		prop_dictionary_get_uint64(npdict, "id-ptr", &itnp);
 		if ((uintptr_t)itnp == (uintptr_t)np) {
 			break;
+		}
+	}
 	if (npdict == NULL) {
 		/* Create NAT policy dictionary and copy the data. */
 		npdict = prop_dictionary_create();
 		npd = prop_data_create_data(np, sizeof(npf_natpolicy_t));
 		prop_dictionary_set(npdict, "nat-policy-data", npd);
 		prop_object_release(npd);
 		CTASSERT(sizeof(uintptr_t) <= sizeof(uint64_t));
 		prop_dictionary_set_uint64(npdict, "id-ptr", (uintptr_t)np);
 		prop_array_add(natlist, npdict);
 		prop_object_release(npdict);
+	}
 	prop_dictionary_set(sedict, "nat-policy", npdict);
 	prop_object_release(npdict);
 	return 0;
+}
 /*
  * npf_nat_restore: find a matching NAT policy and restore NAT entry.
+ *
  * => Caller should lock the active NAT ruleset.
  */
 npf_nat_t *
 npf_nat_restore(prop_dictionary_t sedict, npf_session_t *se)
+{
 	const npf_natpolicy_t *onp;
 	const npf_nat_t *ntraw;
 	prop_object_t obj;
 	npf_natpolicy_t *np;
 	npf_rule_t *rl;
 	npf_nat_t *nt;
 	/* Get raw NAT entry. */
 	obj = prop_dictionary_get(sedict, "nat-data");
 	ntraw = prop_data_data_nocopy(obj);
 	if (ntraw == NULL || prop_data_size(obj) != sizeof(npf_nat_t)) {
 		return NULL;
+	}
 	/* Find a stored NAT policy information. */
 	obj = prop_dictionary_get(
 	    prop_dictionary_get(sedict, "nat-policy"), "nat-policy-data");
 	onp = prop_data_data_nocopy(obj);
 	if (onp == NULL || prop_data_size(obj) != sizeof(npf_natpolicy_t)) {
 		return NULL;
+	}
 	/*
 	 * Match if there is an existing NAT policy.  Will acquire the
 	 * reference on it if further operations are successful.
 	 */
 	KASSERT(npf_config_locked_p());
 	rl = npf_ruleset_matchnat(npf_config_natset(), __UNCONST(onp));
 	if (rl == NULL) {
 		return NULL;
+	}
 	np = npf_rule_getnat(rl);
 	KASSERT(np != NULL);
 	/* Take a specific port from port-map. */
 	if (!npf_nat_takeport(np, ntraw->nt_tport)) {
 		return NULL;
+	}
 	atomic_inc_uint(&np->n_refcnt);
 	/* Create and return NAT entry for association. */
 	nt = pool_cache_get(nat_cache, PR_WAITOK);
 	memcpy(nt, ntraw, sizeof(npf_nat_t));
 	LIST_INSERT_HEAD(&np->n_nat_list, nt, nt_entry);
 	nt->nt_natpolicy = np;
 	nt->nt_session = se;
 	nt->nt_alg = NULL;
 	return nt;
+}
 #if defined(DDB) || defined(_NPF_TESTING)
 void
 npf_nat_dump(const npf_nat_t *nt)
+{
 	const npf_natpolicy_t *np;
 	struct in_addr ip;
 	np = nt->nt_natpolicy;
 	memcpy(&ip, &np->n_taddr, sizeof(ip));
 	printf("\tNATP(%p): type %d flags 0x%x taddr %s tport %d\n",
 	    np, np->n_type, np->n_flags, inet_ntoa(ip), np->n_tport);
 	memcpy(&ip, &nt->nt_oaddr, sizeof(ip));
 	printf("\tNAT: original address %s oport %d tport %d\n",
 	    inet_ntoa(ip), ntohs(nt->nt_oport), ntohs(nt->nt_tport));
 	if (nt->nt_alg) {
 		printf("\tNAT ALG = %p, ARG = %p\n",
 		    nt->nt_alg, (void *)nt->nt_alg_arg);
+	}
+}
 #endif