 @@ -1,14 +1,14 @@
-/*  $NetBSD: ops.c,v 1.6 2010/09/02 08:58:06 manu Exp $ */
+/*  $NetBSD: ops.c,v 1.7 2010/09/03 07:15:18 manu Exp $ */
 /*-
  *  Copyright (c) 2010 Emmanuel Dreyfus. All rights reserved.
+ *
  *  Redistribution and use in source and binary forms, with or without
  *  modification, are permitted provided that the following conditions
  *  are met:
  *  1. Redistributions of source code must retain the above copyright
  *     notice, this list of conditions and the following disclaimer.
  *  2. Redistributions in binary form must reproduce the above copyright
  *     notice, this list of conditions and the following disclaimer in the
  *     documentation and/or other materials provided with the distribution.
+ *
 @@ -31,26 +31,27 @@
 #include <libgen.h>
 #include <errno.h>
 #include <err.h>
 #include <sysexits.h>
 #include <syslog.h>
 #include <puffs.h>
 #include <sys/vnode.h>
 #include <sys/socket.h>
 #include <machine/vmparam.h>
 #include "perfuse_priv.h"
 #include "fuse.h"
 static int node_close_common(struct puffs_usermount *, puffs_cookie_t, int);
 static int no_access(puffs_cookie_t, const struct puffs_cred *, mode_t);
 static void fuse_attr_to_vap(struct perfuse_state *,
     struct vattr *, struct fuse_attr *);
 static int node_lookup_dir_nodot(struct puffs_usermount *,
     puffs_cookie_t, char *, size_t, struct puffs_node **);
 static int node_lookup_common(struct puffs_usermount *, puffs_cookie_t,
     const char*, struct puffs_node **);
 static int node_mk_common(struct puffs_usermount *, puffs_cookie_t,
     struct puffs_newinfo *, const struct puffs_cn *pcn, perfuse_msg_t *);
 static const char *basename_r(const char *);
 static ssize_t fuse_to_dirent(struct puffs_usermount *, puffs_cookie_t,
     struct fuse_dirent *, size_t);
 static int readdir_buffered(struct perfuse_state *, puffs_cookie_t,
 @@ -64,42 +65,111 @@ static void dequeue_requests(struct perf
 /*
  *  From <sys/vnode>, inside #ifdef _KERNEL section
  */
 #define IO_SYNC		(0x40|IO_DSYNC)
 #define IO_DSYNC	0x00200
 #define IO_DIRECT	0x02000
 /*
  *  From <fcntl>, inside #ifdef _KERNEL section
  */
 #define F_WAIT		0x010
 #define F_FLOCK		0x020
 #define OFLAGS(fflags)  ((fflags) - 1)
 /*
  * Borrowed from src/sys/kern/vfs_subr.c and src/sys/sys/vnode.h
  */
 const enum vtype iftovt_tab[16] = {
 	VNON, VFIFO, VCHR, VNON, VDIR, VNON, VBLK, VNON,
         VREG, VNON, VLNK, VNON, VSOCK, VNON, VNON, VBAD,
 };
 const int vttoif_tab[9] = {
 , S_IFREG, S_IFDIR, S_IFBLK, S_IFCHR, S_IFLNK,
         S_IFSOCK, S_IFIFO, S_IFMT,
 };
 #define IFTOVT(mode) (iftovt_tab[((mode) & S_IFMT) >> 12])
 #define VTTOIF(indx) (vttoif_tab[(int)(indx)])
 static int
 node_close_common(pu, opc, mode)
 	struct puffs_usermount *pu;
 	puffs_cookie_t opc;
 	int mode;
+{
 	struct perfuse_state *ps;
 	perfuse_msg_t *pm;
 	int op;
 	uint64_t fh;
 	struct fuse_release_in *fri;
 	struct perfuse_node_data *pnd;
 	struct puffs_node *pn;
 	int error;
 	ps = puffs_getspecific(pu);
 	pn = (struct puffs_node *)opc;
 	pnd = PERFUSE_NODE_DATA(pn);
 	if (puffs_pn_getvap(pn)->va_type == VDIR) {
 		op = FUSE_RELEASEDIR;
 		mode = FREAD;
 	} else {
 		op = FUSE_RELEASE;
+	}
 	/*
 	 * Destroy the filehandle before sending the
 	 * request to the FUSE filesystem, otherwise
 	 * we may get a second close() while we wait
 	 * for the reply, and we would end up closing
 	 * the same fh twice instead of closng both.
 	 */
 	fh = perfuse_get_fh(opc, mode);
 	perfuse_destroy_fh(pn, fh);
 	/*
 	 * release_flags may be set to FUSE_RELEASE_FLUSH
 	 * to flush locks. lock_owner must be set in that case
 	 */
 	pm = ps->ps_new_msg(pu, opc, op, sizeof(*fri), NULL);
 	fri = GET_INPAYLOAD(ps, pm, fuse_release_in);
 	fri->fh = fh;
 	fri->flags = 0;
 	fri->release_flags = 0;
 	fri->lock_owner = pnd->pnd_lock_owner;
 	fri->flags = (fri->lock_owner != 0) ? FUSE_RELEASE_FLUSH : 0;
 #ifdef PERFUSE_DEBUG
 	if (perfuse_diagflags & PDF_FH)
 		DPRINTF("%s: opc = %p, ino = %"PRId64", fh = 0x%"PRIx64"\n",
 			 __func__, (void *)opc, pnd->pnd_ino, fri->fh);
 #endif
 	if ((error = XCHG_MSG(ps, pu, pm, NO_PAYLOAD_REPLY_LEN)) != 0)
 		goto out;
 	ps->ps_destroy_msg(pm);
 	error = 0;
 out:
 	if (error != 0)
 		DERRX(EX_SOFTWARE, "%s: freed fh = 0x%"PRIx64" but filesystem "
 		      "returned error = %d", __func__, fh, error);
 	return error;
+}
 static int
 no_access(opc, pcr, mode)
 	puffs_cookie_t opc;
 	const struct puffs_cred *pcr;
 	mode_t mode;
+{
 	struct puffs_node *pn;
 	struct vattr *va;
 	pn = (struct puffs_node *)opc;
 	va = puffs_pn_getvap(pn);
 @@ -792,26 +862,39 @@ perfuse_fs_suspend(pu, status)
 int
 perfuse_node_lookup(pu, opc, pni, pcn)
 	struct puffs_usermount *pu;
 	puffs_cookie_t opc;
 	struct puffs_newinfo *pni;
 	const struct puffs_cn *pcn;
+{
 	struct puffs_node *pn;
 	int error;
 	/*
 	 * Special case for ..
 	 */
 	if (PCNISDOTDOT(pcn)) {
 		pn = PERFUSE_NODE_DATA(opc)->pnd_parent;
 		PERFUSE_NODE_DATA(pn)->pnd_flags &= ~PND_RECLAIMED;
 		puffs_newinfo_setcookie(pni, pn);
 		puffs_newinfo_setvtype(pni, VDIR);
 		return 0;
+	}
 	/*
 	 * XXX This is borrowed from librefuse,
 	 * and __UNCONST is said to be fixed.
 	 */
         pn = puffs_pn_nodewalk(pu, puffs_path_walkcmp,
         		       __UNCONST(&pcn->pcn_po_full));
 	if (pn == NULL) {
 		error = node_lookup_common(pu, opc, (char *)PCNPATH(pcn), &pn);
 		if (error != 0)
 			return error;
+	}
 	/*
 @@ -856,26 +939,32 @@ perfuse_node_create(pu, opc, pni, pcn, v
 	 * If create is unimplemented: Check that it does not
 	 * already exists, and if not, do mknod and open
 	 */
 	ps = puffs_getspecific(pu);
 	if (ps->ps_flags & PS_NO_CREAT) {
 		error = node_lookup_common(pu, opc, (char*)PCNPATH(pcn), &pn);
 		if (error == 0)
 			return EEXIST;
 		error = perfuse_node_mknod(pu, opc, pni, pcn, vap);
 		if (error != 0)
 			return error;
 		error = node_lookup_common(pu, opc, (char*)PCNPATH(pcn), &pn);
 		if (error != 0)
 			return error;
 		opc = (puffs_cookie_t)pn;
 		error = perfuse_node_open(pu, opc, FREAD|FWRITE, pcn->pcn_cred);
 		if (error != 0)
 			return error;
 		return 0;
+	}
 	name = basename_r((char *)PCNPATH(pcn));
 	namelen = strlen(name) + 1;
 	len = sizeof(*fci) + namelen;
 	pm = ps->ps_new_msg(pu, opc, FUSE_CREATE, len, pcn->pcn_cred);
 	fci = GET_INPAYLOAD(ps, pm, fuse_create_in);
 @@ -888,32 +977,46 @@ perfuse_node_create(pu, opc, pni, pcn, v
 	if ((error = XCHG_MSG(ps, pu, pm, len)) != 0)
 		goto out;
 	feo = GET_OUTPAYLOAD(ps, pm, fuse_entry_out);
 	foo = (struct fuse_open_out *)(void *)(feo + 1);
 	if (feo->nodeid == PERFUSE_UNKNOWN_INO)
 		DERRX(EX_SOFTWARE, "%s: no ino", __func__);
 	/*
 	 * Save the file handle and inode in node private data
 	 * so that we can reuse it later
 	 */
 	pn = perfuse_new_pn(pu, opc);
-	perfuse_new_fh((puffs_cookie_t)pn, foo->fh);
+	perfuse_new_fh((puffs_cookie_t)pn, foo->fh, FWRITE);
 	PERFUSE_NODE_DATA(pn)->pnd_ino = feo->nodeid;
 #ifdef PERFUSE_DEBUG
 	if (perfuse_diagflags & PDF_FH)
 		DPRINTF("%s: opc = %p, file = \"%s\", "
 			"ino = %"PRId64", rfh = 0x%"PRIx64"\n",
 			__func__, (void *)pn, (char *)PCNPATH(pcn),
 			feo->nodeid, foo->fh);
 #endif
 	fuse_attr_to_vap(ps, &pn->pn_va, &feo->attr);
 	puffs_newinfo_setcookie(pni, pn);
 	/*
 	 * It seems we need to do this so that glusterfs gets fully
 	 * aware that the file was created. If we do not do it, we
 	 * get "SETATTR (null) (fuse_loc_fill() failed)"
 	 */
 	(void)node_lookup_common(pu, opc, (char*)PCNPATH(pcn), NULL);
 out:
 	ps->ps_destroy_msg(pm);
 	/*
 	 * create is unimplmented, remember it for later,
 	 * and start over using mknod and open instead.
 	 */
 	if (error == ENOSYS) {
 		ps->ps_flags |= PS_NO_CREAT;
 		return perfuse_node_create(pu, opc, pni, pcn, vap);
+	}
 	return error;
 @@ -969,205 +1072,154 @@ perfuse_node_mknod(pu, opc, pni, pcn, va
 	return node_mk_common(pu, opc, pni, pcn, pm);
+}
 int
 perfuse_node_open(pu, opc, mode, pcr)
 	struct puffs_usermount *pu;
 	puffs_cookie_t opc;
 	int mode;
 	const struct puffs_cred *pcr;
+{
 	struct perfuse_state *ps;
 	struct perfuse_node_data *pnd;
 	perfuse_msg_t *pm;
 	mode_t pmode;
 	mode_t fmode;
 	int op;
 	struct fuse_open_in *foi;
 	struct fuse_open_out *foo;
 	struct puffs_node *pn;
 	int error;
 	ps = puffs_getspecific(pu);
 	pnd = PERFUSE_NODE_DATA(opc);
 	pn = (struct puffs_node *)opc;
 	if (puffs_pn_getvap(pn)->va_type == VDIR) {
 		op = FUSE_OPENDIR;
 		pmode = PUFFS_VREAD|PUFFS_VEXEC;
 	} else {
 		op = FUSE_OPEN;
-		if (mode & (O_RDWR|O_WRONLY))
+		if (mode & FWRITE)
-			pmode = PUFFS_VWRITE;
+			pmode = PUFFS_VWRITE|PUFFS_VREAD;
 		else
 			pmode = PUFFS_VREAD;
+	}
 	/*
 	 * Opening a directory require R-X on the directory
 	 * Opening a file requires R-- for reading, -W- for writing
 	 * In both cases, --X is required on the parent.
 	 */
-	if (no_access((puffs_cookie_t)PERFUSE_NODE_DATA(opc)->pnd_parent,
+	if (no_access((puffs_cookie_t)pnd->pnd_parent, pcr, PUFFS_VEXEC))
 	    pcr, PUFFS_VEXEC))
 		return EACCES;
 	if (no_access(opc, pcr, pmode))
 		return EACCES;
 	/*
 	 * libfuse docs say O_CREAT should not be set.
 	 */
 	mode &= ~O_CREAT;
 	/*
 	 * Do not open twice, and do not reopen for reading
 	 * if we already have write handle.
 	 * Directories are always open with read access only,
 	 * whatever flags we get.
 	 */
 	if (op == FUSE_OPENDIR)
 		mode = (mode & ~(FREAD|FWRITE)) | FREAD;
 	if ((mode & FREAD) && (pnd->pnd_flags & PND_RFH))
 		return 0;
 	if ((mode & FWRITE) && (pnd->pnd_flags & PND_WFH))
 		return 0;
 	/*
 	 * Convert PUFFS mode to FUSE mode: convert FREAD/FWRITE
 	 * to O_RDONLY/O_WRONLY while perserving the other options.
 	 */
 	fmode = mode & ~(FREAD|FWRITE);
 	fmode |= (mode & FWRITE) ? O_RDWR : O_RDONLY;
 	pm = ps->ps_new_msg(pu, opc, op, sizeof(*foi), pcr);
 	foi = GET_INPAYLOAD(ps, pm, fuse_open_in);
-	foi->flags = mode & ~O_ACCMODE;
+	foi->flags = fmode;
 	switch (mode & (FREAD|FWRITE)) {
 	case FREAD|FWRITE:
 		foi->flags |= O_RDWR;
 		break;
 	case FREAD:
 		foi->flags |= O_RDONLY;
 		break;
 	case FWRITE:
 		foi->flags |= O_WRONLY;
 		break;
 	default:
 		break;
 	foi->unused = 0;
 	if ((error = XCHG_MSG(ps, pu, pm, sizeof(*foo))) != 0)
 		goto out;
 	foo = GET_OUTPAYLOAD(ps, pm, fuse_open_out);
 	/*
 	 * Save the file handle in node private data
 	 * so that we can reuse it later
 	 */
-	perfuse_new_fh((puffs_cookie_t)pn, foo->fh);
+	perfuse_new_fh((puffs_cookie_t)pn, foo->fh, mode);
 #ifdef PERFUSE_DEBUG
 	if (perfuse_diagflags & PDF_FH)
 		DPRINTF("%s: opc = %p, file = \"%s\", "
-			"ino = %"PRId64", fh = 0x%"PRIx64"\n",
+			"ino = %"PRId64", %s%sfh = 0x%"PRIx64"\n",
 			__func__, (void *)opc,
 			(char *)PNPATH((struct puffs_node *)opc),
-			PERFUSE_NODE_DATA(opc)->pnd_ino, foo->fh);
+			pnd->pnd_ino, mode & FREAD ? "r" : "",
 			mode & FWRITE ? "w" : "", foo->fh);
 #endif
 out:
 	ps->ps_destroy_msg(pm);
 	return error;
+}
-/* ARGSUSED2 */
+/* ARGSUSED0 */
 int
 perfuse_node_close(pu, opc, flags, pcr)
 	struct puffs_usermount *pu;
 	puffs_cookie_t opc;
 	int flags;
 	const struct puffs_cred *pcr;
+{
 	struct perfuse_state *ps;
 	perfuse_msg_t *pm;
 	int op;
 	uint64_t fh;
 	struct perfuse_node_data *pnd;
 	struct fuse_release_in *fri;
 	struct puffs_node *pn;
-	int error;
+	struct perfuse_node_data *pnd;
 	ps = puffs_getspecific(pu);
 	pn = (struct puffs_node *)opc;
 	pnd = PERFUSE_NODE_DATA(pn);
-	if (puffs_pn_getvap(pn)->va_type == VDIR)
+	pn = (struct puffs_node *)opc;
-		op = FUSE_RELEASEDIR;
+	pnd = PERFUSE_NODE_DATA(opc);
 	else
 		op = FUSE_RELEASE;
 	if (!(pnd->pnd_flags & PND_OPEN))
 		return EBADF;
 	fh = perfuse_get_fh(opc);
 	/*
-	 * Sync before close for files
+	 * Make sure all operation are finished
-	 */
+	 * There can be an ongoing write, or queued operations
-	if ((op == FUSE_RELEASE) && (pnd->pnd_flags & PND_DIRTY)) {
+	 * XXX perhaps deadlock. Use requeue_request
-#ifdef PERFUSE_DEBUG
+	 */
-		if (perfuse_diagflags & PDF_SYNC)
+	while ((pnd->pnd_flags & PND_BUSY) ||
-			DPRINTF("%s: SYNC opc = %p, file = \"%s\"\n",
+	       !TAILQ_EMPTY(&pnd->pnd_pcq))
-				__func__, (void*)opc, (char *)PNPATH(pn));
+		puffs_cc_yield(puffs_cc_getcc(pu));
 #endif
 		if ((error = perfuse_node_fsync(pu, opc, pcr, 0, 0, 0)) != 0)
 			return error;
 		pnd->pnd_flags &= ~PND_DIRTY;
 #ifdef PERFUSE_DEBUG
 		if (perfuse_diagflags & PDF_SYNC)
 			DPRINTF("%s: CLEAR opc = %p, file = \"%s\"\n",
 				__func__, (void*)opc, (char *)PNPATH((pn)));
 #endif
 	/*
 	 * Destroy the filehandle before sending the
 	 * request to the FUSE filesystem, otherwise
 	 * we may get a second close() while we wait
 	 * for the reply, and we would end up closing
 	 * the same fh twice instead of closng both.
 	 */
 	perfuse_destroy_fh(pn, fh);
 #ifdef PERFUSE_DEBUG
 	if (perfuse_diagflags & PDF_FH)
 		DPRINTF("%s: opc = %p, ino = %"PRId64", fh = 0x%"PRIx64"\n",
 			__func__, (void *)opc, pnd->pnd_ino, fh);
 #endif
 	/*
-	 * release_flags may be set to FUSE_RELEASE_FLUSH
+	 * The NetBSD kernel will send sync and setattr(mtime, ctime)
-	 * to flush locks. lock_owner must be set in that case
+	 * afer a close on a regular file. Some FUSE filesystem will
 	 * assume theses operations are performed on open files. We
 	 * therefore postpone the close operation at reclaim time.
 	 */
-	pm = ps->ps_new_msg(pu, opc, op, sizeof(*fri), NULL);
+	if (puffs_pn_getvap(pn)->va_type != VREG)
-	fri = GET_INPAYLOAD(ps, pm, fuse_release_in);
+		return node_close_common(pu, opc, flags);
 	fri->fh = fh;
 	fri->flags = 0;
 	fri->release_flags = 0;
 	fri->lock_owner = PERFUSE_NODE_DATA(pn)->pnd_lock_owner;
 	fri->flags = (fri->lock_owner != 0) ? FUSE_RELEASE_FLUSH : 0;
 #ifdef PERFUSE_DEBUG
 	if (perfuse_diagflags & PDF_FH)
 		DPRINTF("%s: opc = %p, ino = %"PRId64", fh = 0x%"PRIx64"\n",
 			 __func__, (void *)opc, pnd->pnd_ino, fri->fh);
 #endif
 	if ((error = XCHG_MSG(ps, pu, pm, NO_PAYLOAD_REPLY_LEN)) != 0)
 		goto out;
 out:
 	if (error != 0)
 		DWARNX("%s: freed fh = 0x%"PRIx64" but filesystem "
 		       "returned error = %d",
 		       __func__, fh, error);
-	ps->ps_destroy_msg(pm);
+	return 0;
 	return error;
+}
 int
 perfuse_node_access(pu, opc, mode, pcr)
 	struct puffs_usermount *pu;
 	puffs_cookie_t opc;
 	int mode;
 	const struct puffs_cred *pcr;
+{
 	perfuse_msg_t *pm;
 	struct perfuse_state *ps;
 	struct fuse_access_in *fai;
 	int error;
 @@ -1190,27 +1242,27 @@ perfuse_node_access(pu, opc, mode, pcr)
+	}
 	if (error == ENOSYS) {
 		struct fuse_getattr_in *fgi;
 		struct fuse_attr_out *fao;
 		ps->ps_flags |= PS_NO_ACCESS;
 		pm = ps->ps_new_msg(pu, opc, FUSE_GETATTR,
 			      sizeof(*fgi), NULL);
 		fgi = GET_INPAYLOAD(ps, pm, fuse_getattr_in);
 		fgi->getattr_flags = 0;
 		fgi->dummy = 0;
-		fgi->fh = perfuse_get_fh(opc);
+		fgi->fh = perfuse_get_fh(opc, FREAD);
 #ifdef PERFUSE_DEBUG
 		if (perfuse_diagflags & PDF_FH)
 			DPRINTF("%s: opc = %p, ino = %"PRId64", "
 				"fh = 0x%"PRIx64"\n", __func__, (void *)opc,
 				PERFUSE_NODE_DATA(opc)->pnd_ino, fgi->fh);
 #endif
 		if ((error = XCHG_MSG(ps, pu, pm, sizeof(*fao))) != 0) {
 			ps->ps_destroy_msg(pm);
 			goto out;
+		}
 		fao = GET_OUTPAYLOAD(ps, pm, fuse_attr_out);
 @@ -1278,35 +1330,40 @@ out:
 	return error;
+}
 int
 perfuse_node_setattr(pu, opc, vap, pcr)
 	struct puffs_usermount *pu;
 	puffs_cookie_t opc;
 	const struct vattr *vap;
 	const struct puffs_cred *pcr;
+{
 	perfuse_msg_t *pm;
 	uint64_t fh;
 	struct perfuse_state *ps;
 	struct perfuse_node_data *pnd;
 	struct fuse_setattr_in *fsi;
 	int error;
 	int open_self;
 	struct vattr *old_va;
 	open_self = 0;
 	ps = puffs_getspecific(pu);
 	pnd = PERFUSE_NODE_DATA(opc);
 	/*
 	 * setattr requires --X on the parent directory
 	 */
-	if (no_access((puffs_cookie_t)PERFUSE_NODE_DATA(opc)->pnd_parent,
+	if (no_access((puffs_cookie_t)pnd->pnd_parent, pcr, PUFFS_VEXEC))
 	    pcr, PUFFS_VEXEC))
 		return EACCES;
 	old_va = puffs_pn_getvap((struct puffs_node *)opc);
 	/*
 	 * Check for permission to change size
 	 */
 	if ((vap->va_size != (u_quad_t)PUFFS_VNOVAL) &&
 	    no_access(opc, pcr, PUFFS_VWRITE))
 		return EACCES;
 	/*
 	 * Check for permission to change dates
 @@ -1321,41 +1378,58 @@ perfuse_node_setattr(pu, opc, vap, pcr)
 	 * Check for permission to change owner and group
 	 */
 	if (((vap->va_uid != (uid_t)PUFFS_VNOVAL) ||
 	     (vap->va_gid != (gid_t)PUFFS_VNOVAL)) &&
 	    (puffs_access_chown(old_va->va_uid, old_va->va_gid,
 				vap->va_uid, vap->va_gid, pcr)) != 0)
 		return EACCES;
 	/*
 	 * Check for permission to change permissions
 	 */
 	if ((vap->va_mode != (mode_t)PUFFS_VNOVAL) &&
 	    (puffs_access_chmod(old_va->va_uid, old_va->va_gid,
-				 old_va->va_type, vap->va_mode, pcr)) != 0)
+				old_va->va_type, vap->va_mode, pcr)) != 0)
 		return EACCES;
 	/*
 	 * setattr(mtime, ctime) require an open file,
 	 * at least for glusterfs.
 	 */
 	if (((vap->va_atime.tv_sec != (time_t)PUFFS_VNOVAL) ||
 	     (vap->va_mtime.tv_sec != (time_t)PUFFS_VNOVAL)) &&
 	    !(pnd->pnd_flags & PND_WFH)) {
 		if ((error = perfuse_node_open(pu, opc, FWRITE, pcr)) != 0)
 			return error;
 		open_self = 1;
+	}
 	/*
 	 * It seems troublesome to resize a file while
 	 * a write is just beeing done. Wait for
 	 * it to finish.
 	 */
 	if (vap->va_size != (u_quad_t)PUFFS_VNOVAL)
 		while (pnd->pnd_flags & PND_INWRITE)
 			requeue_request(pu, opc, PCQ_AFTERWRITE);
 	ps = puffs_getspecific(pu);
 	pm = ps->ps_new_msg(pu, opc, FUSE_SETATTR, sizeof(*fsi), pcr);
 	fsi = GET_INPAYLOAD(ps, pm, fuse_setattr_in);
 	fsi->valid = 0;
-	if (PERFUSE_NODE_DATA(opc)->pnd_flags & PND_OPEN) {
+	if (pnd->pnd_flags & PND_WFH) {
-		fh = perfuse_get_fh(opc);
+		fh = perfuse_get_fh(opc, FWRITE);
 		fsi->fh = fh;
-		if (fh != FUSE_UNKNOWN_FH)
+		fsi->valid |= FUSE_FATTR_FH;
 			fsi->valid |= FUSE_FATTR_FH;
+	}
 	if (vap->va_size != (u_quad_t)PUFFS_VNOVAL) {
 		fsi->size = vap->va_size;
 		fsi->valid |= FUSE_FATTR_SIZE;
+	}
 	if (vap->va_atime.tv_sec != (time_t)PUFFS_VNOVAL) {
 		fsi->atime = vap->va_atime.tv_sec;;
 		fsi->atimensec = (uint32_t)vap->va_atime.tv_nsec;;
 		fsi->valid |= (FUSE_FATTR_ATIME|FUSE_FATTR_ATIME_NOW);
+	}
 @@ -1370,60 +1444,63 @@ perfuse_node_setattr(pu, opc, vap, pcr)
 		fsi->valid |= FUSE_FATTR_MODE;
+	}
 	if (vap->va_uid != (uid_t)PUFFS_VNOVAL) {
 		fsi->uid = vap->va_uid;
 		fsi->valid |= FUSE_FATTR_UID;
+	}
 	if (vap->va_gid != (gid_t)PUFFS_VNOVAL) {
 		fsi->gid = vap->va_gid;
 		fsi->valid |= FUSE_FATTR_GID;
+	}
-	if (PERFUSE_NODE_DATA(opc)->pnd_lock_owner != 0) {
+	if (pnd->pnd_lock_owner != 0) {
-		fsi->lock_owner = PERFUSE_NODE_DATA(opc)->pnd_lock_owner;
+		fsi->lock_owner = pnd->pnd_lock_owner;
 		fsi->valid |= FUSE_FATTR_LOCKOWNER;
+	}
 	/*
 	 * A fuse_attr_out is returned, but we ignore it.
 	 */
 	error = XCHG_MSG(ps, pu, pm, sizeof(struct fuse_attr_out));
 	ps->ps_destroy_msg(pm);
 	if (open_self)
 		(void)perfuse_node_close(pu, opc, FWRITE, pcr);
 	return error;
+}
 int
 perfuse_node_poll(pu, opc, events)
 	struct puffs_usermount *pu;
 	puffs_cookie_t opc;
 	int *events;
+{
 	struct perfuse_state *ps;
 	perfuse_msg_t *pm;
 	struct fuse_poll_in *fpi;
 	struct fuse_poll_out *fpo;
 	int error;
 	ps = puffs_getspecific(pu);
 	/*
 	 * kh is set if FUSE_POLL_SCHEDULE_NOTIFY is set.
  	 */
 	pm = ps->ps_new_msg(pu, opc, FUSE_POLL, sizeof(*fpi), NULL);
 	fpi = GET_INPAYLOAD(ps, pm, fuse_poll_in);
-	fpi->fh = perfuse_get_fh(opc);
+	fpi->fh = perfuse_get_fh(opc, FREAD);
 	fpi->kh = 0;
 	fpi->flags = 0;
 #ifdef PERFUSE_DEBUG
 	if (perfuse_diagflags & PDF_FH)
 		DPRINTF("%s: opc = %p, ino = %"PRId64", fh = 0x%"PRIx64"\n",
 			__func__, (void *)opc,
 			PERFUSE_NODE_DATA(opc)->pnd_ino, fpi->fh);
 #endif
 	if ((error = XCHG_MSG(ps, pu, pm, sizeof(*fpo))) != 0)
 		goto out;
 	fpo = GET_OUTPAYLOAD(ps, pm, fuse_poll_out);
 @@ -1469,51 +1546,51 @@ perfuse_node_fsync(pu, opc, pcr, flags,
 	pm = NULL;
 	open_self = 0;
 	/*
 	 * If we previously detected it as unimplemented,
 	 * skip the call to the filesystem.
 	 */
 	ps = puffs_getspecific(pu);
 	if (ps->ps_flags == PS_NO_FSYNC)
 		return ENOSYS;
 	/*
 	 * Do not sync if there are no change to sync
-	 * XXX remove that testif we implement mmap
+	 * XXX remove that test if we implement mmap
 	 */
 	pnd = PERFUSE_NODE_DATA(opc);
 #ifdef PERFUSE_DEBUG
 	if (perfuse_diagflags & PDF_SYNC)
 		DPRINTF("%s: TEST opc = %p, file = \"%s\" is %sdirty\n",
 			__func__, (void*)opc,
 			(char *)PNPATH((struct puffs_node *)opc),
 			pnd->pnd_flags & PND_DIRTY ? "" : "not ");
 #endif
 	if (!(pnd->pnd_flags & PND_DIRTY))
 		return 0;
 	/*
 	 * It seems NetBSD can call fsync without open first
 	 * glusterfs complain in such a situation:
 	 * "FSYNC() ERR => -1 (Invalid argument)"
 	 */
 	if (!(pnd->pnd_flags & PND_OPEN)) {
-		if ((error = perfuse_node_open(pu, opc, FREAD, pcr)) != 0)
+		if ((error = perfuse_node_open(pu, opc, FWRITE, pcr)) != 0)
 			goto out;
 		open_self = 1;
+	}
-	fh = perfuse_get_fh(opc);
+	fh = perfuse_get_fh(opc, FWRITE);
 	/*
 	 * If fsync_flags  is set, meta data should not be flushed.
 	 */
 	pm = ps->ps_new_msg(pu, opc, FUSE_FSYNC, sizeof(*ffi), NULL);
 	ffi = GET_INPAYLOAD(ps, pm, fuse_fsync_in);
 	ffi->fh = fh;
 	ffi->fsync_flags = (flags & FFILESYNC) ? 0 : 1;
 #ifdef PERFUSE_DEBUG
 	if (perfuse_diagflags & PDF_FH)
 		DPRINTF("%s: opc = %p, ino = %"PRId64", fh = 0x%"PRIx64"\n",
 			__func__, (void *)opc,
 @@ -1533,28 +1610,28 @@ perfuse_node_fsync(pu, opc, pcr, flags,
 	if (perfuse_diagflags & PDF_SYNC)
 		DPRINTF("%s: CLEAR opc = %p, file = \"%s\"\n",
 			__func__, (void*)opc,
 			(char *)PNPATH((struct puffs_node *)opc));
 #endif
 out:
 	if (error == ENOSYS)
 		ps->ps_flags |= PS_NO_FSYNC;
 	if (pm != NULL)
 		ps->ps_destroy_msg(pm);
 	if (open_self)
-		(void)perfuse_node_close(pu, opc, 0, pcr);
+		(void)node_close_common(pu, opc, FWRITE);
 	return error;
+}
 /* ARGSUSED0 */
 int
 perfuse_node_seek(pu, opc, oldoff, newoff,  pcr)
 	struct puffs_usermount *pu;
 	puffs_cookie_t opc;
 	off_t oldoff;
 	off_t newoff;
 	const struct puffs_cred *pcr;
+{
 @@ -1566,63 +1643,62 @@ perfuse_node_seek(pu, opc, oldoff, newof
 	PERFUSE_NODE_DATA(opc)->pnd_offset = newoff;
 	return 0;
+}
 int
 perfuse_node_remove(pu, opc, targ, pcn)
 	struct puffs_usermount *pu;
 	puffs_cookie_t opc;
 	puffs_cookie_t targ;
 	const struct puffs_cn *pcn;
+{
 	struct perfuse_state *ps;
 	perfuse_msg_t *pm;
 	struct puffs_node *pn;
 	struct perfuse_node_data *pnd;
 	perfuse_msg_t *pm;
 	char *path;
 	const char *name;
 	size_t len;
 	int error;
 	pnd = PERFUSE_NODE_DATA(opc);
 	/*
 	 * remove requires -WX on the parent directory
 	 * no right required on the object.
 	 */
-	if (no_access((puffs_cookie_t)PERFUSE_NODE_DATA(opc)->pnd_parent,
+	if (no_access((puffs_cookie_t)pnd->pnd_parent,
 	    pcn->pcn_cred, PUFFS_VWRITE|PUFFS_VEXEC))
 		return EACCES;
 	ps = puffs_getspecific(pu);
 	if (targ == NULL)
 		DERRX(EX_SOFTWARE, "%s: targ is NULL", __func__);
 	ps = puffs_getspecific(pu);
 	pn = (struct puffs_node *)targ;
 	name = basename_r((char *)PNPATH(pn));
 	len = strlen(name) + 1;
 	pm = ps->ps_new_msg(pu, opc, FUSE_UNLINK, len, pcn->pcn_cred);
 	path = _GET_INPAYLOAD(ps, pm, char *);
 	(void)strlcpy(path, name, len);
 	if ((error = XCHG_MSG(ps, pu, pm, UNSPEC_REPLY_LEN)) != 0)
 		goto out;
 	if (puffs_inval_namecache_dir(pu, opc) != 0)
 		DERR(EX_OSERR, "puffs_inval_namecache_dir failed");
 	if (puffs_inval_pagecache_node(pu, (puffs_cookie_t)pn) != 0)
 		DERR(EX_OSERR, "puffs_inval_namecache_node failed");
 	puffs_setback(puffs_cc_getcc(pu), PUFFS_SETBACK_NOREF_N2);
 	/*
 	 * Reclaim should take care of decreasing pnd_childcount
 	 */
 out:
 	ps->ps_destroy_msg(pm);
 	return error;
+}
 int
 perfuse_node_link(pu, opc, targ, pcn)
 @@ -1757,59 +1833,59 @@ perfuse_node_mkdir(pu, opc, pni, pcn, va
 	return node_mk_common(pu, opc, pni, pcn, pm);
+}
 int
 perfuse_node_rmdir(pu, opc, targ, pcn)
 	struct puffs_usermount *pu;
 	puffs_cookie_t opc;
 	puffs_cookie_t targ;
 	const struct puffs_cn *pcn;
+{
 	struct perfuse_state *ps;
 	struct perfuse_node_data *pnd;
 	perfuse_msg_t *pm;
 	struct puffs_node *pn;
 	char *path;
 	const char *name;
 	size_t len;
 	int error;
 	pnd = PERFUSE_NODE_DATA(opc);
 	/*
 	 * remove requires -WX on the parent directory
 	 * no right required on the object.
 	 */
-	if (no_access((puffs_cookie_t)PERFUSE_NODE_DATA(opc)->pnd_parent,
+	if (no_access((puffs_cookie_t)pnd->pnd_parent,
 	    pcn->pcn_cred, PUFFS_VWRITE|PUFFS_VEXEC))
 		return EACCES;
 	ps = puffs_getspecific(pu);
 	pn = (struct puffs_node *)targ;
 	name = basename_r((char *)PNPATH(pn));
 	len = strlen(name) + 1;
 	pm = ps->ps_new_msg(pu, opc, FUSE_RMDIR, len, pcn->pcn_cred);
 	path = _GET_INPAYLOAD(ps, pm, char *);
 	(void)strlcpy(path, name, len);
 	if ((error = XCHG_MSG(ps, pu, pm, UNSPEC_REPLY_LEN)) != 0)
 		goto out;
 	if (puffs_inval_namecache_dir(pu, opc) != 0)
 		DERR(EX_OSERR, "puffs_inval_namecache_dir failed");
 	if (puffs_inval_pagecache_node(pu, (puffs_cookie_t)pn) != 0)
 		DERR(EX_OSERR, "puffs_inval_namecache_node failed");
 	puffs_setback(puffs_cc_getcc(pu), PUFFS_SETBACK_NOREF_N2);
 out:
 	ps->ps_destroy_msg(pm);
 	return error;
+}
 /* vap is unused */
 /* ARGSUSED4 */
 int
 perfuse_node_symlink(pu, opc, pni, pcn_src, vap, link_target)
 	struct puffs_usermount *pu;
 @@ -1895,37 +1971,37 @@ perfuse_node_readdir(pu, opc, dent, read
 			__func__, (void *)opc);
 #endif
 	/*
 	 * Do we already have the data bufered?
 	 */
 	if (pnd->pnd_dirent != NULL)
 		goto out;
 	pnd->pnd_dirent_len = 0;
 	/*
 	 * It seems NetBSD can call readdir without open first
 	 * libfuse will crash if it is done that way, hence open first.
 	 */
-	if (!(PERFUSE_NODE_DATA(opc)->pnd_flags & PND_OPEN)) {
+	if (!(pnd->pnd_flags & PND_OPEN)) {
 		if ((error = perfuse_node_open(pu, opc, FREAD, pcr)) != 0)
 			goto out;
 		open_self = 1;
+	}
-	fh = perfuse_get_fh(opc);
+	fh = perfuse_get_fh(opc, FREAD);
 #ifdef PERFUSE_DEBUG
 	if (perfuse_diagflags & PDF_FH)
-		DPRINTF("%s: opc = %p, ino = %"PRId64", fh = 0x%"PRIx64"\n",
+		DPRINTF("%s: opc = %p, ino = %"PRId64", rfh = 0x%"PRIx64"\n",
 			__func__, (void *)opc,
 			PERFUSE_NODE_DATA(opc)->pnd_ino, fh);
 #endif
 	pnd->pnd_all_fd = NULL;
 	pnd->pnd_all_fd_len = 0;
 	fd_offset = 0;
 	do {
 		size_t fd_len;
 		char *afdp;
 		pm = ps->ps_new_msg(pu, opc, FUSE_READDIR, sizeof(*fri), pcr);
 @@ -2003,27 +2079,27 @@ out:
 		free(pnd->pnd_all_fd);
 		pnd->pnd_all_fd = NULL;
 		pnd->pnd_all_fd_len = 0;
+	}
 	if (pm != NULL)
 		ps->ps_destroy_msg(pm);
 	/*
 	 * If we opened the directory ourselves, close now
 	 * errors are ignored.
 	 */
 	if (open_self)
-		(void)perfuse_node_close(pu, opc, 0, pcr);
+		(void)perfuse_node_close(pu, opc, FWRITE, pcr);
 	if (error == 0)
 		error = readdir_buffered(ps, opc, dent, readoff,
 			reslen, pcr, eofflag, cookies, ncookies);
 	/*
 	 * Schedule queued readdir requests
 	 */
 	dequeue_requests(ps, opc, PCQ_READDIR, DEQUEUE_ALL);
 	pnd->pnd_flags &= ~PND_INREADDIR;
 #ifdef PERFUSE_DEBUG
 	if (perfuse_diagflags & PDF_READDIR)
 @@ -2082,77 +2158,99 @@ perfuse_node_reclaim(pu, opc)
 	puffs_cookie_t opc;
+{
 	struct perfuse_state *ps;
 	perfuse_msg_t *pm;
 	struct perfuse_node_data *pnd;
 	struct fuse_forget_in *ffi;
 	struct puffs_node *pn;
 	struct puffs_node *pn_root;
 	ps = puffs_getspecific(pu);
 	pnd = PERFUSE_NODE_DATA(opc);
 	/*
 	 * Make sure open files are properly closed when reclaimed.
 	 */
 	while (pnd->pnd_flags & PND_OPEN)
 		(void)perfuse_node_close(pu, opc, 0, NULL);
 	/*
 	 * Never forget the root.
 	 */
 	if (pnd->pnd_ino == FUSE_ROOT_ID)
 		return 0;
 	pnd->pnd_flags |= PND_RECLAIMED;
 #ifdef PERFUSE_DEBUG
 	if (perfuse_diagflags & PDF_RECLAIM)
 		DPRINTF("%s (nodeid %"PRId64") reclaimed\n",
 		       (char *)PNPATH((struct puffs_node *)opc), pnd->pnd_ino);
 #endif
 	pn_root = puffs_getroot(pu);
 	pn = (struct puffs_node *)opc;
 	while (pn != pn_root) {
 		struct puffs_node *parent_pn;
 		pnd = PERFUSE_NODE_DATA(pn);
 #ifdef PERFUSE_DEBUG
 	if (perfuse_diagflags & PDF_RECLAIM)
 		DPRINTF("%s (nodeid %"PRId64") is %sreclaimed, "
-			"has childcount %d, %sopen\n",
+			"has childcount %d %s%s%s, pending ops:%s%s%s%s\n",
 		        (char *)PNPATH(pn), pnd->pnd_ino,
 		        pnd->pnd_flags & PND_RECLAIMED ? "" : "not ",
 		        pnd->pnd_childcount,
-			pnd->pnd_flags & PND_OPEN ? "" : "not ");
+			pnd->pnd_flags & PND_OPEN ? "open " : "not open",
 			pnd->pnd_flags & PND_RFH ? "r" : "",
-	if (pnd->pnd_flags & PND_OPEN)
+			pnd->pnd_flags & PND_WFH ? "w" : "",
-		DWARNX("%s: (nodeid %"PRId64") %s is still open",
+			pnd->pnd_flags & PND_INREADDIR ? " readdir" : "",
-		       __func__, pnd->pnd_ino, (char *)PNPATH(pn));
+			pnd->pnd_flags & PND_INREAD ? " read" : "",
 			pnd->pnd_flags & PND_INWRITE ? " write" : "",
 			pnd->pnd_flags & PND_BUSY ? "" : " none");
 #endif
 		if (!(pnd->pnd_flags & PND_RECLAIMED) ||
 		    (pnd->pnd_childcount != 0))
 			return 0;
 		/*
-		 * If the file is still open, close all file handles
+		 * Make sure all operation are finished
-		 * XXX no pcr arguement to send.
+		 * There can be an ongoing write, or queued operations
 		 */
-		while(pnd->pnd_flags & PND_OPEN)
+		while (pnd->pnd_flags & PND_INWRITE) {
-			(void)perfuse_node_close(pu, opc, 0, NULL);
+			requeue_request(pu, opc, PCQ_AFTERWRITE);
 			/*
 			 * It may have been cancelled in the meantime
 			 */
 			if (!(pnd->pnd_flags & PND_RECLAIMED))
 				return 0;
+		}
 #ifdef PERFUSE_DEBUG
 		if ((pnd->pnd_flags & PND_BUSY) ||
 		       !TAILQ_EMPTY(&pnd->pnd_pcq))
 			DERRX(EX_SOFTWARE, "%s: opc = %p: ongoing operations",
 			      __func__, (void *)opc);
 #endif
 		/*
 		 * Close open files
 		 */
 		if (pnd->pnd_flags & PND_WFH)
 			(void)node_close_common(pu, opc, FREAD);
 		if (pnd->pnd_flags & PND_RFH)
 			(void)node_close_common(pu, opc, FWRITE);
 		/*
 		 * And send the FORGET message
 		 */
 		pm = ps->ps_new_msg(pu, (puffs_cookie_t)pn, FUSE_FORGET,
 			      sizeof(*ffi), NULL);
 		ffi = GET_INPAYLOAD(ps, pm, fuse_forget_in);
 		ffi->nlookup = pnd->pnd_nlookup;
 		/*
 		 * No reply is expected, pm is freed in XCHG_MSG
 		 */
 		(void)XCHG_MSG_NOREPLY(ps, pu, pm, UNSPEC_REPLY_LEN);
 		parent_pn = pnd->pnd_parent;
 		perfuse_destroy_pn(pn);
 @@ -2213,27 +2311,27 @@ perfuse_node_advlock(pu, opc, id, op, fl
 	struct fuse_lk_in *fli;
 	struct fuse_lk_out *flo;
 	int error;
 	ps = puffs_getspecific(pu);
 	if (op == F_GETLK)
 		fop = FUSE_GETLK;
 	else
 		fop = (flags & F_WAIT) ? FUSE_SETLKW : FUSE_SETLK;
 	pm = ps->ps_new_msg(pu, opc, fop, sizeof(*fli), NULL);
 	fli = GET_INPAYLOAD(ps, pm, fuse_lk_in);
-	fli->fh = perfuse_get_fh(opc);
+	fli->fh = perfuse_get_fh(opc, FWRITE);
 	fli->owner = fl->l_pid;
 	fli->lk.start = fl->l_start;
 	fli->lk.end = fl->l_start + fl->l_len;
 	fli->lk.type = fl->l_type;
 	fli->lk.pid = fl->l_pid;
 	fli->lk_flags = (flags & F_FLOCK) ? FUSE_LK_FLOCK : 0;
 #ifdef PERFUSE_DEBUG
 	if (perfuse_diagflags & PDF_FH)
 		DPRINTF("%s: opc = %p, ino = %"PRId64", fh = 0x%"PRIx64"\n",
 			__func__, (void *)opc,
 			PERFUSE_NODE_DATA(opc)->pnd_ino, fli->fh);
 #endif
 @@ -2269,67 +2367,73 @@ out:
+}
 int
 perfuse_node_read(pu, opc, buf, offset, resid, pcr, ioflag)
 	struct puffs_usermount *pu;
 	puffs_cookie_t opc;
 	uint8_t *buf;
 	off_t offset;
 	size_t *resid;
 	const struct puffs_cred *pcr;
 	int ioflag;
+{
 	struct perfuse_state *ps;
 	struct perfuse_node_data *pnd;
 	perfuse_msg_t *pm;
 	struct fuse_read_in *fri;
 	struct fuse_out_header *foh;
 	size_t readen;
 	size_t requested;
 	int error;
 	ps = puffs_getspecific(pu);
 	pnd = PERFUSE_NODE_DATA(opc);
 	pm = NULL;
 	if (puffs_pn_getvap((struct puffs_node *)opc)->va_type == VDIR)
 		return EBADF;
 	pnd->pnd_flags |= PND_INREAD;
 	requested = *resid;
 	if ((ps->ps_readahead + requested) > ps->ps_max_readahead) {
 		if (perfuse_diagflags & PDF_REQUEUE)
 			DPRINTF("readahead = %zd\n", ps->ps_readahead);
 		requeue_request(pu, opc, PCQ_READ);
+	}
 	ps->ps_readahead += requested;
 	do {
 		/*
 		 * flags may be set to FUSE_READ_LOCKOWNER
 		 * if lock_owner is provided.
+		 *
 		 * XXX See comment about fri->size in perfuse_node_readdir
 		 * We encounter the same bug here.
 		 */
 		pm = ps->ps_new_msg(pu, opc, FUSE_READ, sizeof(*fri), pcr);
 		fri = GET_INPAYLOAD(ps, pm, fuse_read_in);
-		fri->fh = perfuse_get_fh(opc);
+		fri->fh = perfuse_get_fh(opc, FREAD);
 		fri->offset = offset;
 		fri->size = (uint32_t)MIN(*resid, PAGE_SIZE - sizeof(*foh));
 		fri->read_flags = 0; /* XXX Unused by libfuse? */
-		fri->lock_owner = PERFUSE_NODE_DATA(opc)->pnd_lock_owner;
+		fri->lock_owner = pnd->pnd_lock_owner;
 		fri->flags = 0;
 		fri->flags |= (fri->lock_owner != 0) ? FUSE_READ_LOCKOWNER : 0;
 #ifdef PERFUSE_DEBUG
 	if (perfuse_diagflags & PDF_FH)
 		DPRINTF("%s: opc = %p, ino = %"PRId64", fh = 0x%"PRIx64"\n",
-			__func__, (void *)opc,
+			__func__, (void *)opc, pnd->pnd_ino, fri->fh);
 			PERFUSE_NODE_DATA(opc)->pnd_ino, fri->fh);
 #endif
 		error = XCHG_MSG(ps, pu, pm, UNSPEC_REPLY_LEN);
 		if (error  != 0)
 			goto out;
 		foh = GET_OUTHDR(ps, pm);
 		readen = foh->len - sizeof(*foh);
 		(void)memcpy(buf,  _GET_OUTPAYLOAD(ps, pm, char *), readen);
 		buf += readen;
 		offset += readen;
 @@ -2341,93 +2445,102 @@ perfuse_node_read(pu, opc, buf, offset,
 	if (ioflag & (IO_SYNC|IO_DSYNC))
 		ps->ps_syncreads++;
 	else
 		ps->ps_asyncreads++;
 out:
 	if (pm != NULL)
 		ps->ps_destroy_msg(pm);
 	ps->ps_readahead -= requested;
 	dequeue_requests(ps, opc, PCQ_READ, 1);
 	pnd->pnd_flags &= ~PND_INREAD;
 	return error;
+}
 int
 perfuse_node_write(pu, opc, buf, offset, resid, pcr, ioflag)
 	struct puffs_usermount *pu;
 	puffs_cookie_t opc;
 	uint8_t *buf;
 	off_t offset;
 	size_t *resid;
 	const struct puffs_cred *pcr;
 	int ioflag;
+{
 	struct perfuse_state *ps;
 	struct perfuse_node_data *pnd;
 	perfuse_msg_t *pm;
 	struct fuse_write_in *fwi;
 	struct fuse_write_out *fwo;
 	size_t data_len;
 	size_t payload_len;
 	size_t written;
 	size_t requested;
 	int error;
 	ps = puffs_getspecific(pu);
 	pnd = PERFUSE_NODE_DATA(opc);
 	pm = NULL;
 	written = 0;
 	if (puffs_pn_getvap((struct puffs_node *)opc)->va_type == VDIR)
 		return EBADF;
 DPRINTF("%s ENTER\n", __func__);
 	pnd->pnd_flags |= PND_INWRITE;
 	requested = *resid;
 	if ((ps->ps_write + requested) > ps->ps_max_write) {
 		if (perfuse_diagflags & PDF_REQUEUE)
 			DPRINTF("write = %zd\n", ps->ps_write);
 		requeue_request(pu, opc, PCQ_WRITE);
+	}
 	ps->ps_write += requested;
 	do {
 		/*
 		 * It seems libfuse does not expects big chunks, so
 		 * send it page per page. The writepage feature is
 		 * probably there to minmize data movement.
 		 * XXX use ps->ps_maxwrite?
 		 */
 		data_len = MIN(*resid, PAGE_SIZE);
 		payload_len = data_len + sizeof(*fwi);
 		/*
 		 * flags may be set to FUSE_WRITE_CACHE (XXX usage?)
 		 * or FUSE_WRITE_LOCKOWNER, if lock_owner is provided.
 		 * write_flags is set to 1 for writepage.
 		 */
 		pm = ps->ps_new_msg(pu, opc, FUSE_WRITE, payload_len, pcr);
 		fwi = GET_INPAYLOAD(ps, pm, fuse_write_in);
-		fwi->fh = perfuse_get_fh(opc);
+		fwi->fh = perfuse_get_fh(opc, FWRITE);
 		fwi->offset = offset;
 		fwi->size = (uint32_t)data_len;
 		fwi->write_flags = (fwi->size % PAGE_SIZE) ? 0 : 1;
-		fwi->lock_owner = PERFUSE_NODE_DATA(opc)->pnd_lock_owner;
+		fwi->lock_owner = pnd->pnd_lock_owner;
 		fwi->flags = 0;
 		fwi->flags |= (fwi->lock_owner != 0) ? FUSE_WRITE_LOCKOWNER : 0;
 		fwi->flags |= (ioflag & IO_DIRECT) ? 0 : FUSE_WRITE_CACHE;
 		(void)memcpy((fwi + 1), buf + written, data_len);
 #ifdef PERFUSE_DEBUG
 	if (perfuse_diagflags & PDF_FH)
 		DPRINTF("%s: opc = %p, ino = %"PRId64", fh = 0x%"PRIx64"\n",
-			__func__, (void *)opc,
+			__func__, (void *)opc, pnd->pnd_ino, fwi->fh);
 			PERFUSE_NODE_DATA(opc)->pnd_ino, fwi->fh);
 #endif
 		if ((error = XCHG_MSG(ps, pu, pm, sizeof(*fwo))) != 0)
 			goto out;
 		fwo = GET_OUTPAYLOAD(ps, pm, fuse_write_out);
 		written = fwo->size;
 		*resid -= written;
 		offset += written;
 		buf += written;
 		ps->ps_destroy_msg(pm);
 		pm = NULL;
 	} while (*resid != 0);
 @@ -2437,41 +2550,48 @@ perfuse_node_write(pu, opc, buf, offset,
 	 *  "everything must be written or an error will be generated"
 	 */
 	if (*resid != 0)
 		error = EFBIG;
 	if (ioflag & (IO_SYNC|IO_DSYNC))
 		ps->ps_syncwrites++;
 	else
 		ps->ps_asyncwrites++;
 	/*
 	 * Remember to sync the file
 	 */
-	PERFUSE_NODE_DATA(opc)->pnd_flags |= PND_DIRTY;
+	pnd->pnd_flags |= PND_DIRTY;
 #ifdef PERFUSE_DEBUG
 	if (perfuse_diagflags & PDF_SYNC)
 		DPRINTF("%s: DIRTY opc = %p, file = \"%s\"\n",
 			__func__, (void*)opc,
 			(char *)PNPATH((struct puffs_node *)opc));
 #endif
 out:
 	if (pm != NULL)
 		ps->ps_destroy_msg(pm);
 	ps->ps_write -= requested;
 	dequeue_requests(ps, opc, PCQ_WRITE, 1);
 	pnd->pnd_flags &= ~PND_INWRITE;
 	/*
 	 * Dequeue operation that were waiting for write to complete
 	 */
 	dequeue_requests(ps, opc, PCQ_AFTERWRITE, DEQUEUE_ALL);
 	return error;
+}
 /* ARGSUSED0 */
 void
 perfuse_cache_write(pu, opc, size, runs)
 	struct puffs_usermount *pu;
 	puffs_cookie_t opc;
 	size_t size;
 	struct puffs_cacherun *runs;
+{
 	return;
+}

 @@ -1,14 +1,14 @@
-/*  $NetBSD: perfuse_priv.h,v 1.4 2010/09/01 14:57:24 manu Exp $ */
+/*  $NetBSD: perfuse_priv.h,v 1.5 2010/09/03 07:15:18 manu Exp $ */
 /*-
  *  Copyright (c) 2010 Emmanuel Dreyfus. All rights reserved.
+ *
  *  Redistribution and use in source and binary forms, with or without
  *  modification, are permitted provided that the following conditions
  *  are met:
  *  1. Redistributions of source code must retain the above copyright
  *     notice, this list of conditions and the following disclaimer.
  *  2. Redistributions in binary form must reproduce the above copyright
  *     notice, this list of conditions and the following disclaimer in the
  *     documentation and/or other materials provided with the distribution.
+ *
 @@ -61,57 +61,58 @@ struct perfuse_state {
 	uint64_t ps_unique;
 	size_t ps_readahead;
 	size_t ps_write;
 	perfuse_new_msg_fn ps_new_msg;
 	perfuse_xchg_msg_fn ps_xchg_msg;
 	perfuse_destroy_msg_fn ps_destroy_msg;
 	perfuse_get_inhdr_fn ps_get_inhdr;
 	perfuse_get_inpayload_fn ps_get_inpayload;
 	perfuse_get_outhdr_fn ps_get_outhdr;
 	perfuse_get_outpayload_fn ps_get_outpayload;
 };
-struct perfuse_file_handle {
+enum perfuse_qtype { PCQ_READDIR, PCQ_READ, PCQ_WRITE, PCQ_AFTERWRITE };
 	uint64_t pfh_fh;
 	TAILQ_ENTRY(perfuse_file_handle) pfh_entries;
 };
 enum perfuse_qtype { PCQ_READDIR, PCQ_READ, PCQ_WRITE };
 struct perfuse_cc_queue {
 	enum perfuse_qtype pcq_type;
 	struct puffs_cc *pcq_cc;
 	TAILQ_ENTRY(perfuse_cc_queue) pcq_next;
 };
 struct perfuse_node_data {
-	TAILQ_HEAD(,perfuse_file_handle) pnd_fh;
+	uint64_t pnd_rfh;
 	uint64_t pnd_wfh;
 	uint64_t pnd_ino;			/* inode */
 	uint64_t pnd_nlookup;			/* vnode refcount */
 	uint64_t pnd_offset;			/* seek state */
 	uint64_t pnd_lock_owner;
 	struct dirent *pnd_dirent;		/* native buffer for readdir */
 	size_t pnd_dirent_len;
 	struct fuse_dirent *pnd_all_fd;		/* FUSE buffer for readdir */
 	size_t pnd_all_fd_len;
 	TAILQ_HEAD(,perfuse_cc_queue) pnd_pcq;	/* queued requests */
 	int pnd_flags;
-#define PND_RECLAIMED		0x1	/* reclaim pending */
+#define PND_RECLAIMED		0x01	/* reclaim pending */
-#define PND_INREADDIR		0x2	/* readdir in progress */
+#define PND_INREADDIR		0x02	/* readdir in progress */
-#define PND_OPEN		0x4	/* At least one fh is allocated */
+#define PND_DIRTY		0x04	/* There is some data to sync */
-#define PND_DIRTY		0x8	/* There is some data to sync */
+#define PND_RFH			0x08	/* Read FH allocated */
 #define PND_WFH			0x10	/* Write FH allocated */
 #define PND_INREAD		0x20	/* read in progress */
 #define PND_INWRITE		0x40	/* write in progress */
 #define PND_OPEN		(PND_RFH|PND_WFH)	/* File is open */
 #define PND_BUSY		(PND_INREADDIR|PND_INREAD|PND_INWRITE)
 	puffs_cookie_t pnd_parent;
 	int pnd_childcount;
 };
 #define PERFUSE_NODE_DATA(opc)	\
 	((struct perfuse_node_data *)puffs_pn_getpriv((struct puffs_node *)opc))
 #define UNSPEC_REPLY_LEN PERFUSE_UNSPEC_REPLY_LEN /* shorter! */
 #define NO_PAYLOAD_REPLY_LEN 0
 #define GET_INHDR(ps, pm) ps->ps_get_inhdr(pm)
 #define GET_INPAYLOAD(ps, pm, type) \
 @@ -121,29 +122,29 @@ struct perfuse_node_data {
 #define GET_OUTPAYLOAD(ps, pm, type) \
 	(struct type *)(void *)ps->ps_get_outpayload(pm)
 #define _GET_OUTPAYLOAD(ps, pm, type) (type)ps->ps_get_outpayload(pm)
 #define XCHG_MSG(ps, pu, opc, len) ps->ps_xchg_msg(pu, opc, len, wait_reply)
 #define XCHG_MSG_NOREPLY(ps, pu, opc, len) \
     ps->ps_xchg_msg(pu, opc, len, no_reply)
 __BEGIN_DECLS
 struct puffs_node *perfuse_new_pn(struct puffs_usermount *,
     struct puffs_node *);
 void perfuse_destroy_pn(struct puffs_node *);
-void perfuse_new_fh(puffs_cookie_t, uint64_t);
+void perfuse_new_fh(puffs_cookie_t, uint64_t, int);
 void perfuse_destroy_fh(puffs_cookie_t, uint64_t);
-uint64_t perfuse_get_fh(puffs_cookie_t);
+uint64_t perfuse_get_fh(puffs_cookie_t, int);
 uint64_t perfuse_next_unique(struct puffs_usermount *);
 char *perfuse_fs_mount(int, ssize_t);
 /*
  * opc.c - filesystem operations
  */
 int perfuse_fs_unmount(struct puffs_usermount *, int);
 int perfuse_fs_statvfs(struct puffs_usermount *, struct statvfs *);
 int perfuse_fs_sync(struct puffs_usermount *, int,
     const struct puffs_cred *);
 int perfuse_fs_fhtonode(struct puffs_usermount *, void *, size_t,

 @@ -1,14 +1,14 @@
-/*  $NetBSD: subr.c,v 1.3 2010/09/01 14:57:24 manu Exp $ */
+/*  $NetBSD: subr.c,v 1.4 2010/09/03 07:15:18 manu Exp $ */
 /*-
  *  Copyright (c) 2010 Emmanuel Dreyfus. All rights reserved.
+ *
  *  Redistribution and use in source and binary forms, with or without
  *  modification, are permitted provided that the following conditions
  *  are met:
  *  1. Redistributions of source code must retain the above copyright
  *     notice, this list of conditions and the following disclaimer.
  *  2. Redistributions in binary form must reproduce the above copyright
  *     notice, this list of conditions and the following disclaimer in the
  *     documentation and/or other materials provided with the distribution.
+ *
 @@ -41,27 +41,28 @@ perfuse_new_pn(pu, parent)
 	struct puffs_usermount *pu;
 	struct puffs_node *parent;
+{
 	struct puffs_node *pn;
 	struct perfuse_node_data *pnd;
 	if ((pnd = malloc(sizeof(*pnd))) == NULL)
 		DERR(EX_OSERR, "malloc failed");
 	if ((pn = puffs_pn_new(pu, pnd)) == NULL)
 		DERR(EX_SOFTWARE, "puffs_pn_new failed");
 	(void)memset(pnd, 0, sizeof(*pnd));
-	TAILQ_INIT(&pnd->pnd_fh);
+	pnd->pnd_rfh = FUSE_UNKNOWN_FH;
 	pnd->pnd_wfh = FUSE_UNKNOWN_FH;
 	pnd->pnd_ino = PERFUSE_UNKNOWN_INO;
 	pnd->pnd_nlookup = 1;
 	pnd->pnd_parent = parent;
 	TAILQ_INIT(&pnd->pnd_pcq);
 	if (parent != NULL)
 		PERFUSE_NODE_DATA(parent)->pnd_childcount++;
 	return pn;
+}
 void
 perfuse_destroy_pn(pn)
 @@ -69,98 +70,112 @@ perfuse_destroy_pn(pn)
+{
 	struct perfuse_node_data *pnd;
 	if ((pnd = puffs_pn_getpriv(pn)) != NULL) {
 		if (pnd->pnd_parent != NULL)
 			PERFUSE_NODE_DATA(pnd->pnd_parent)->pnd_childcount--;
 		if (pnd->pnd_dirent != NULL)
 			free(pnd->pnd_dirent);
 		if (pnd->pnd_all_fd != NULL)
 			free(pnd->pnd_all_fd);
 #ifdef PERFUSE_DEBUG
-		if (!TAILQ_EMPTY(&pnd->pnd_fh))
+		if (pnd->pnd_flags & PND_OPEN)
-			DERRX(EX_SOFTWARE, "%s: non empty pnd_fh", __func__);
+			DERRX(EX_SOFTWARE, "%s: file open", __func__);
 		if (!TAILQ_EMPTY(&pnd->pnd_pcq))
 			DERRX(EX_SOFTWARE, "%s: non empty pnd_pcq", __func__);
 #endif /* PERFUSE_DEBUG */
 		free(pnd);
+	}
 	puffs_pn_remove(pn);
 	return;
+}
 void
-perfuse_new_fh(opc, fh)
+perfuse_new_fh(opc, fh, mode)
 	puffs_cookie_t opc;
 	uint64_t fh;
 	int mode;
+{
 	struct perfuse_node_data *pnd;
 	struct perfuse_file_handle *pfh;
 	if (fh == FUSE_UNKNOWN_FH)
 		return;
 	pnd = PERFUSE_NODE_DATA(opc);
 	pnd->pnd_flags |= PND_OPEN;
 	if ((pfh = malloc(sizeof(*pfh))) == NULL)
 		DERR(EX_OSERR, "malloc failed");
 	pfh->pfh_fh = fh;
-	TAILQ_INSERT_TAIL(&pnd->pnd_fh, pfh, pfh_entries);
+	if (mode & FWRITE) {
 		if (pnd->pnd_flags & PND_WFH)
 			DERRX(EX_SOFTWARE, "%s: opc = %p, write fh already set",
 			      __func__, (void *)opc);
 		pnd->pnd_wfh = fh;
 		pnd->pnd_flags |= PND_WFH;
+	}
 	if (mode & FREAD) {
 		if (pnd->pnd_flags & PND_RFH)
 			DERRX(EX_SOFTWARE, "%s: opc = %p, read fh already set",
 			      __func__, (void *)opc);
 		pnd->pnd_rfh = fh;
 		pnd->pnd_flags |= PND_RFH;
+	}
 	return;
+}
 void
 perfuse_destroy_fh(opc, fh)
 	puffs_cookie_t opc;
 	uint64_t fh;
+{
 	struct perfuse_node_data *pnd;
 	struct perfuse_file_handle *pfh;
 	pnd = PERFUSE_NODE_DATA(opc);
-	TAILQ_FOREACH(pfh, &pnd->pnd_fh, pfh_entries) {
+	if (fh == pnd->pnd_rfh) {
-		if (pfh->pfh_fh == fh) {
+		if (!(pnd->pnd_flags & PND_RFH) && (fh != FUSE_UNKNOWN_FH))
-			TAILQ_REMOVE(&pnd->pnd_fh, pfh, pfh_entries);
+			DERRX(EX_SOFTWARE,
-			free(pfh);
+			      "%s: opc = %p, unset rfh = %"PRIx64"",
-			break;
+			      __func__, (void *)opc, fh);
 		pnd->pnd_rfh = FUSE_UNKNOWN_FH;
 		pnd->pnd_flags &= ~PND_RFH;
+	}
-	if (TAILQ_EMPTY(&pnd->pnd_fh))
+	if (fh == pnd->pnd_wfh) {
-		pnd->pnd_flags &= ~PND_OPEN;
+		if (!(pnd->pnd_flags & PND_WFH) && (fh != FUSE_UNKNOWN_FH))
 			DERRX(EX_SOFTWARE,
 			      "%s: opc = %p, unset wfh = %"PRIx64"",
 			      __func__, (void *)opc, fh);
 		pnd->pnd_wfh = FUSE_UNKNOWN_FH;
 		pnd->pnd_flags &= ~PND_WFH;
+	}
 	if (pfh == NULL)
 		DERRX(EX_SOFTWARE,
 		      "%s: unexistant fh = %"PRId64" (double close?)",
 		      __func__, fh);
 	return;
+}
 uint64_t
-perfuse_get_fh(opc)
+perfuse_get_fh(opc, mode)
 	puffs_cookie_t opc;
 	int mode;
+{
 	struct perfuse_node_data *pnd;
 	struct perfuse_file_handle *pfh;
 	uint64_t fh = FUSE_UNKNOWN_FH;
 	pnd = PERFUSE_NODE_DATA(opc);
-	if ((pfh = TAILQ_FIRST(&pnd->pnd_fh)) != NULL)
+	if (mode & FWRITE)
-		fh = pfh->pfh_fh;;
+		if (pnd->pnd_flags & PND_WFH)
 			return pnd->pnd_wfh;
 	if (mode & FREAD) {
 		if (pnd->pnd_flags & PND_RFH)
 			return pnd->pnd_rfh;
 		if (pnd->pnd_flags & PND_WFH)
 			return pnd->pnd_wfh;
+	}
-	return fh;
+	return FUSE_UNKNOWN_FH;
+}