From 15e83fec25056029bc016d220687e353cab285f8 Mon Sep 17 00:00:00 2001
From: David van Moolenbroek <david@minix3.org>
Date: Wed, 11 Sep 2013 00:50:36 +0200
Subject: [PATCH] LWIP: move chardev message parsing into libnetsock

Change-Id: Ie23fd003c9fa35811548f388c8e9b55e8d9de8d7
---
 include/minix/netsock.h    |  60 ++++----
 lib/libnetsock/socket.c    | 192 +++++++++++++-----------
 servers/lwip/driver.c      | 227 +++++++++++++----------------
 servers/lwip/eth.c         |   1 +
 servers/lwip/inet_config.c |   3 +-
 servers/lwip/lwip.c        |  27 ++--
 servers/lwip/proto.h       |   4 +-
 servers/lwip/raw_ip.c      | 146 ++++++++-----------
 servers/lwip/tcp.c         | 290 ++++++++++++++++++-------------------
 servers/lwip/udp.c         | 132 ++++++++---------
 10 files changed, 520 insertions(+), 562 deletions(-)

diff --git a/include/minix/netsock.h b/include/minix/netsock.h
index c341cc2fd..0c553e066 100644
--- a/include/minix/netsock.h
+++ b/include/minix/netsock.h
@@ -5,6 +5,7 @@
 
 #include <minix/ipc.h>
 #include <minix/endpoint.h>
+#include <minix/chardriver.h>
 
 /*
  * User can set this variable to make the debugging output differ between
@@ -18,19 +19,21 @@ extern char * netsock_user_name;
 #define SOCK_TYPES	3
 
 struct socket;
+struct sock_req;
 
-typedef void (* sock_op_t)(struct socket *, message *);
-typedef void (* sock_op_io_t)(struct socket *, message *, int blk);
-typedef int (* sock_op_open_t)(struct socket *, message *);
+typedef int (* sock_op_opcl_t)(struct socket *);
+typedef int (* sock_op_io_t)(struct socket *, struct sock_req *, int blk);
+typedef int (* sock_op_select_t)(struct socket *, unsigned int);
+typedef int (* sock_op_select_reply_t)(struct socket *);
 
 struct sock_ops {
-	sock_op_open_t	open;
-	sock_op_t	close;
-	sock_op_io_t	read;
-	sock_op_io_t	write;
-	sock_op_io_t	ioctl;
-	sock_op_t	select;
-	sock_op_t	select_reply;
+	sock_op_opcl_t		open;
+	sock_op_opcl_t		close;
+	sock_op_io_t		read;
+	sock_op_io_t		write;
+	sock_op_io_t		ioctl;
+	sock_op_select_t	select;
+	sock_op_select_reply_t	select_reply;
 };
 
 struct recv_q {
@@ -63,6 +66,23 @@ struct recv_q {
 						SOCK_FLG_SEL_ERROR);	\
 } while (0)
 
+struct sock_req {
+	enum {
+		SOCK_REQ_READ,
+		SOCK_REQ_WRITE,
+		SOCK_REQ_IOCTL
+	}			type;
+	devminor_t		minor;
+	endpoint_t		endpt;
+	cp_grant_id_t		grant;
+	union {
+		size_t		size;	/* for SOCK_REQ_READ, SOCK_REQ_WRITE */
+		unsigned long	req;	/* for SOCK_REQ_IOCTL */
+	};
+	int			flags;
+	cdev_id_t		id;
+};
+
 struct socket {
 	int			type;
 	u32_t			flags;
@@ -71,9 +91,7 @@ struct socket {
 	struct sock_ops *	ops;
 	void *			buf;
 	size_t			buf_size;
-	message			mess; /* store the message which initiated the
-					 last operation on this socket in case
-					 we have to suspend the operation */
+	struct sock_req		req;
 	void *			shm;
 	size_t			shm_size;
 	endpoint_t		select_ep;
@@ -87,7 +105,7 @@ struct socket {
  * Each component needs to provide a method how to initially open a socket.
  * The rest is handled byt the socket library.
  */
-void socket_open(message * m);
+int socket_open(devminor_t minor);
 
 #define get_sock_num(x) ((long int) ((x) - socket))
 #define is_valid_sock_num(x) (x < MAX_SOCKETS)
@@ -103,16 +121,10 @@ extern struct socket socket[MAX_SOCKETS];
 void socket_request(message * m);
 void mq_process(void);
 
-
 struct socket * get_unused_sock(void);
 struct socket * get_nic_sock(unsigned dev);
 
-void send_reply(message * m, int status);
-void send_reply_open(message * m, int status);
-void send_reply_close(message * m, int status);
-void sock_reply(struct socket * sock, int status);
-void sock_reply_close(struct socket * sock, int status);
-void sock_reply_select(struct socket * sock, unsigned selops);
+void send_req_reply(struct sock_req * req, int status);
 
 typedef void (* recv_data_free_fn)(void *);
 
@@ -139,10 +151,8 @@ static inline void * debug_malloc(size_t s)
 	free(x);								\
 } while(0)
 
-void generic_op_select(struct socket * sock, message * m);
-void generic_op_select_reply(struct socket * sock, message * m);
-
-int mq_enqueue(message * m);
+int generic_op_select(struct socket * sock, unsigned int sel);
+int generic_op_select_reply(struct socket * sock);
 
 /* a function thr user has to provide to reply to the posix server */
 void posix_reply(endpoint_t ep, message * m);
diff --git a/lib/libnetsock/socket.c b/lib/libnetsock/socket.c
index 7897a126e..74d6482e2 100644
--- a/lib/libnetsock/socket.c
+++ b/lib/libnetsock/socket.c
@@ -46,7 +46,7 @@ struct socket socket[MAX_SOCKETS];
 #define recv_q_free	debug_free
 
 struct mq {
-	message		m;
+	struct sock_req	req;
 	struct mq *	prev;
 	struct mq *	next;
 };
@@ -56,18 +56,18 @@ struct mq {
 
 static struct mq * mq_head, *mq_tail;
 
-int mq_enqueue(message * m)
+int mq_enqueue(struct sock_req * req)
 {
 	struct mq * mq;
 
-	debug_sock_print("sock %d op %d", m->DEVICE, m->m_type);
+	debug_sock_print("sock %d op %d", req->minor, req->type);
 	mq = mq_alloc();
 
 	if (mq == NULL)
 		return -1;
 
 	mq->next = NULL;
-	mq->m = *m;
+	mq->req = *req;
 
 	if (mq_head) {
 		mq->prev = mq_tail;
@@ -97,7 +97,7 @@ __unused static struct mq * mq_dequeue_head(void)
 	} else
 		mq_head = mq_tail = NULL;
 
-	debug_sock_print("socket %d\n", ret->m.DEVICE);
+	debug_sock_print("socket %d\n", ret->req.minor);
 
 	return ret;
 }
@@ -120,15 +120,14 @@ static void mq_dequeue(struct mq * mq)
 	}
 }
 
-static int mq_cancel(message * m)
+static int mq_cancel(devminor_t minor, endpoint_t endpt, cdev_id_t id)
 {
 	struct mq * mq;
 
 	for (mq = mq_tail; mq; mq = mq->prev) {
-		if (m->DEVICE == mq->m.DEVICE &&
-				m->USER_ENDPT == mq->m.USER_ENDPT &&
-				m->IO_GRANT == mq->m.IO_GRANT) {
-			debug_sock_print("socket %d\n", mq->m.DEVICE);
+		if (minor == mq->req.minor && endpt == mq->req.endpt &&
+				id == mq->req.id) {
+			debug_sock_print("socket %d\n", minor);
 			break;
 		}
 	}
@@ -138,6 +137,7 @@ static int mq_cancel(message * m)
 		mq_free(mq);
 	}
 
+	/* FIXME: shouldn't this return (!!mq) ? */
 	return 1;
 }
 
@@ -203,7 +203,7 @@ static void set_reply_msg(message * m, int status)
 	m->REP_IO_GRANT= ref;
 }
 
-void send_reply_type(message * m, int type, int status)
+static void send_reply_type(message * m, int type, int status)
 {
 	int result;
 
@@ -215,25 +215,44 @@ void send_reply_type(message * m, int type, int status)
 		netsock_panic("unable to send (err %d)", result);
 }
 
-void send_reply(message * m, int status)
+void send_req_reply(struct sock_req * req, int status)
+{
+	message m;
+	int result;
+
+	if (status == EDONTREPLY)
+		return;
+
+	m.m_type = DEV_REVIVE;
+	m.REP_STATUS = status;
+	m.REP_ENDPT = req->endpt; /* FIXME: HACK */
+	m.REP_IO_GRANT = req->id;
+
+	result = send(req->endpt, &m);
+	if (result != OK)
+		netsock_panic("unable to send (err %d)", result);
+}
+
+static void send_reply(message * m, int status)
 {
 	debug_sock_print("status %d", status);
 	send_reply_type(m, DEV_REVIVE, status);
 }
 
-void send_reply_open(message * m, int status)
+static void send_reply_open(message * m, int status)
 {
 	debug_sock_print("status %d", status);
 	send_reply_type(m, DEV_OPEN_REPL, status);
 }
 
-void send_reply_close(message * m, int status)
+static void send_reply_close(message * m, int status)
 {
 	debug_sock_print("status %d", status);
 	send_reply_type(m, DEV_CLOSE_REPL, status);
 }
 
-void sock_reply_select(struct socket * sock, unsigned selops)
+static void sock_reply_select(struct socket * sock, endpoint_t endpt,
+	unsigned selops)
 {
 	int result;
 	message msg;
@@ -244,7 +263,7 @@ void sock_reply_select(struct socket * sock, unsigned selops)
 	msg.DEV_MINOR = get_sock_num(sock);
 	msg.DEV_SEL_OPS = selops;
 
-	result = send(sock->select_ep, &msg);
+	result = send(endpt, &msg);
 	if (result != OK)
 		netsock_panic("unable to send (err %d)", result);
 }
@@ -257,8 +276,7 @@ void sock_select_notify(struct socket * sock)
 	debug_sock_select_print("socket num %ld", get_sock_num(sock));
 	assert(sock->select_ep != NONE);
 
-	msg.DEV_SEL_OPS = 0;
-	sock->ops->select_reply(sock, &msg);
+	msg.DEV_SEL_OPS = sock->ops->select_reply(sock);
 	if (msg.DEV_SEL_OPS == 0) {
 		debug_sock_select_print("called from %p sflags 0x%x TXsz %d RXsz %d\n",
 				__builtin_return_address(0), sock->flags,
@@ -279,25 +297,6 @@ void sock_select_notify(struct socket * sock)
 	sock->select_ep = NONE;
 }
 
-static void sock_reply_type(struct socket * sock, int type, int status)
-{
-	sock->mess.m_type = type;
-
-	send_reply_type(&sock->mess, type, status);
-}
-
-void sock_reply_close(struct socket * sock, int status)
-{
-	debug_sock_print("sock %ld status %d", get_sock_num(sock), status);
-	sock_reply_type(sock, DEV_CLOSE_REPL, status);
-}
-
-void sock_reply(struct socket * sock, int status)
-{
-	debug_sock_print("sock %ld status %d", get_sock_num(sock), status);
-	sock_reply_type(sock, DEV_REVIVE, status);
-}
-
 struct socket * get_unused_sock(void)
 {
 	int i;
@@ -313,51 +312,56 @@ struct socket * get_unused_sock(void)
 	return NULL;
 }
 
-static void socket_request_socket(struct socket * sock, message * m)
+static int socket_request_socket(struct socket * sock, struct sock_req * req)
 {
-	int blocking = m->FLAGS & FLG_OP_NONBLOCK ? 0 : 1;
+	int r, blocking = req->flags & FLG_OP_NONBLOCK ? 0 : 1;
 
-	switch (m->m_type) {
-	case DEV_READ_S:
+	switch (req->type) {
+	case SOCK_REQ_READ:
 		if (sock->ops && sock->ops->read)
-			sock->ops->read(sock, m, blocking);
+			r = sock->ops->read(sock, req, blocking);
 		else
-			send_reply(m, EINVAL);
-		return;
-	case DEV_WRITE_S:
+			r = EINVAL;
+		break;
+	case SOCK_REQ_WRITE:
 		if (sock->ops && sock->ops->write)
-			sock->ops->write(sock, m, blocking);
+			r = sock->ops->write(sock, req, blocking);
 		else
-			send_reply(m, EINVAL);
-		return;
-	case DEV_IOCTL_S:
+			r = EINVAL;
+		break;
+	case SOCK_REQ_IOCTL:
 		if (sock->ops && sock->ops->ioctl)
-			sock->ops->ioctl(sock, m, blocking);
+			r = sock->ops->ioctl(sock, req, blocking);
 		else
-			send_reply(m, EINVAL);
-		return;
+			r = EINVAL;
+		break;
 	default:
 		netsock_panic("cannot happen!");
 	}
+
+	return r;
 }
 
 void socket_request(message * m)
 {
 	struct socket * sock;
+	struct sock_req req;
+	int r;
 
 	debug_sock_print("request %d", m->m_type);
 	switch (m->m_type) {
 	case DEV_OPEN:
-		socket_open(m);
+		r = socket_open(m->DEVICE);
+		send_reply_open(m, r);
 		return;
 	case DEV_CLOSE:
 		sock = get_sock(m->DEVICE);
 		if (sock->ops && sock->ops->close) {
 			sock->flags &= ~SOCK_FLG_OP_PENDING;
-			sock->mess = *m;
-			sock->ops->close(sock, m);
+			r = sock->ops->close(sock);
 		} else
-			send_reply_close(m, EINVAL);
+			r = EINVAL;
+		send_reply_close(m, r);
 		return;
 	case DEV_READ_S:
 	case DEV_WRITE_S:
@@ -367,6 +371,24 @@ void socket_request(message * m)
 			send_reply(m, EINVAL);
 			return;
 		}
+		/* Build a request record for this request. */
+		req.minor = m->DEVICE;
+		req.endpt = m->m_source;
+		req.grant = (cp_grant_id_t) m->IO_GRANT;
+		req.id = (cdev_id_t) m->IO_GRANT;
+		req.flags = m->FLAGS;
+		switch (m->m_type) {
+		case DEV_READ_S:
+		case DEV_WRITE_S:
+			req.type = (m->m_type == DEV_READ_S) ?
+				SOCK_REQ_READ : SOCK_REQ_WRITE;
+			req.size = m->COUNT;
+			break;
+		case DEV_IOCTL_S:
+			req.type = SOCK_REQ_IOCTL;
+			req.req = m->REQUEST;
+			break;
+		}
 		/*
 		 * If an operation is pending (blocking operation) or writing is
 		 * still going and we want to read, suspend the new operation
@@ -383,33 +405,32 @@ void socket_request(message * m)
 				o = "non R/W op";
 			debug_sock_print("socket %ld is busy by %s flgs 0x%x\n",
 					get_sock_num(sock), o, sock->flags);
-			if (mq_enqueue(m) != 0) {
+			if (mq_enqueue(&req) != 0) {
 				debug_sock_print("Enqueuing suspended "
 							"call failed");
 				send_reply(m, ENOMEM);
 			}
 			return;
 		}
-		sock->mess = *m;
-		socket_request_socket(sock, m);
+		sock->req = req;
+		r = socket_request_socket(sock, &req);
+		send_req_reply(&req, r);
 		return;
 	case CANCEL:
 		sock = get_sock(m->DEVICE);
 		printf("socket num %ld\n", get_sock_num(sock));
 		debug_sock_print("socket num %ld", get_sock_num(sock));
 		/* Cancel the last operation in the queue */
-		if (mq_cancel(m)) {
+		if (mq_cancel(m->DEVICE, m->m_source,
+				(cdev_id_t) m->IO_GRANT)) {
 			send_reply(m, EINTR);
-			return;
 		/* ... or a blocked read */
 		} else if (sock->flags & SOCK_FLG_OP_PENDING &&
 				sock->flags & SOCK_FLG_OP_READING) {
 			sock->flags &= ~SOCK_FLG_OP_PENDING;
 			send_reply(m, EINTR);
-			return;
-		} else
-			netsock_panic("no operation to cancel");
-
+		}
+		/* The request may not be found. This is OK. Do not reply. */
 		return;
 	case DEV_SELECT:
 		/*
@@ -421,11 +442,13 @@ void socket_request(message * m)
 
 		if (sock->ops && sock->ops->select) {
 			sock->select_ep = m->m_source;
-			sock->ops->select(sock, m);
+			r = sock->ops->select(sock, m->DEV_SEL_OPS);
 			if (!sock_select_set(sock))
 				sock->select_ep = NONE;
 		} else
-			send_reply(m, EINVAL);
+			r = EINVAL;
+
+		sock_reply_select(sock, m->m_source, r);
 		return;
 	default:
 		netsock_error("unknown message from VFS, type %d\n",
@@ -438,20 +461,22 @@ void mq_process(void)
 {
 	struct mq * mq;
 	struct socket * sock;
+	int r;
 
 	mq = mq_head;
 
 	while(mq) {
 		struct mq * next = mq->next;
 
-		sock = get_sock(mq->m.DEVICE);
+		sock = get_sock(mq->req.minor);
 		if (!(sock->flags & SOCK_FLG_OP_PENDING) &&
-				!(mq->m.m_type == DEV_READ_S &&
+				!(mq->req.type == SOCK_REQ_READ &&
 					sock->flags & SOCK_FLG_OP_WRITING)) {
 			debug_sock_print("resuming op on sock %ld\n",
 					get_sock_num(sock));
-			sock->mess = mq->m;
-			socket_request_socket(sock, &sock->mess);
+			sock->req = mq->req;
+			r = socket_request_socket(sock, &sock->req);
+			send_req_reply(&sock->req, r);
 			mq_dequeue(mq);
 			mq_free(mq);
 			return;
@@ -461,13 +486,11 @@ void mq_process(void)
 	}
 }
 
-void generic_op_select(struct socket * sock, message * m)
+int generic_op_select(struct socket * sock, unsigned int sel)
 {
-	int retsel = 0, sel;
+	int retsel = 0;
 
-	debug_sock_print("socket num %ld 0x%x", get_sock_num(sock), m->USER_ENDPT);
-
-	sel = m->USER_ENDPT;
+	debug_sock_print("socket num %ld 0x%x", get_sock_num(sock), sel);
 
 	/* in this case any operation would block, no error */
 	if (sock->flags & SOCK_FLG_OP_PENDING) {
@@ -478,8 +501,7 @@ void generic_op_select(struct socket * sock, message * m)
 				sock->flags |= SOCK_FLG_SEL_WRITE;
 			/* FIXME we do not monitor error */
 		}
-		sock_reply_select(sock, 0);
-		return;
+		return 0;
 	}
 
 	if (sel & SEL_RD) {
@@ -493,11 +515,13 @@ void generic_op_select(struct socket * sock, message * m)
 		retsel |= SEL_WR;
 	/* FIXME SEL_ERR is ignored, we do not generate exceptions */
 
-	sock_reply_select(sock, retsel);
+	return retsel;
 }
 
-void generic_op_select_reply(struct socket * sock, __unused message * m)
+int generic_op_select_reply(struct socket * sock)
 {
+	unsigned int sel = 0;
+
 	assert(sock->select_ep != NONE);
 	debug_sock_print("socket num %ld", get_sock_num(sock));
 
@@ -506,13 +530,15 @@ void generic_op_select_reply(struct socket * sock, __unused message * m)
 
 	if (sock->flags & SOCK_FLG_OP_PENDING) {
 		debug_sock_print("WARNING socket still blocking!");
-		return;
+		return 0;
 	}
 
 	if (sock->flags & SOCK_FLG_SEL_READ && sock->recv_head)
-		m->DEV_SEL_OPS |= SEL_RD;
+		sel |= SEL_RD;
 
-	if (m->DEV_SEL_OPS)
+	if (sel)
 		sock->flags &= ~(SOCK_FLG_SEL_WRITE | SOCK_FLG_SEL_READ |
 							SOCK_FLG_SEL_ERROR);
+
+	return sel;
 }
diff --git a/servers/lwip/driver.c b/servers/lwip/driver.c
index 6bd3f1a76..2bf1baafc 100644
--- a/servers/lwip/driver.c
+++ b/servers/lwip/driver.c
@@ -258,11 +258,11 @@ __unused static void print_pkt(unsigned char * pkt, int len)
 	printf("--- PKT END ---\n");
 }
 
-static int raw_receive(message * m,
+static int raw_receive(struct sock_req *req,
 			struct pbuf *pbuf)
 {
 	struct pbuf * p;
-	unsigned int rem_len = m->COUNT;
+	unsigned int rem_len = req->size;
 	unsigned int written = 0;
 	int err;
 
@@ -272,10 +272,8 @@ static int raw_receive(message * m,
 		size_t cp_len;
 
 		cp_len = (rem_len < p->len) ? rem_len : p->len;
-		err = copy_to_user(m->m_source,	p->payload, cp_len,
-				(cp_grant_id_t) m->IO_GRANT,
-				written);
-
+		err = copy_to_user(req->endpt, p->payload, cp_len,
+				   req->grant, written);
 		if (err != OK)
 			return err;
 
@@ -300,16 +298,13 @@ int raw_socket_input(struct pbuf * pbuf, struct nic * nic)
 	if (sock->flags & SOCK_FLG_OP_PENDING) {
 		int ret;
 		/* we are resuming a suspended operation */
-		ret = raw_receive(&sock->mess, pbuf);
+		ret = raw_receive(&sock->req, pbuf);
 
-		if (ret > 0) {
-			sock_reply(sock, ret);
-			sock->flags &= ~SOCK_FLG_OP_PENDING;
+		send_req_reply(&sock->req, ret);
+		sock->flags &= ~SOCK_FLG_OP_PENDING;
+
+		if (ret > 0)
 			return 0;
-		} else {
-			sock_reply(sock, ret);
-			sock->flags &= ~SOCK_FLG_OP_PENDING;
-		}
 	}
 
 	/* Do not enqueue more data than allowed */
@@ -453,11 +448,11 @@ static void raw_recv_free(__unused void * data)
 	pbuf_free((struct pbuf *) data);
 }
 
-static void nic_op_close(struct socket * sock, __unused message * m)
+static int nic_op_close(struct socket * sock)
 {
 	struct nic * nic = (struct nic *)sock->data;
 
-	debug_drv_print("socket %d", get_sock_num(sock));
+	debug_drv_print("socket %ld", get_sock_num(sock));
 	
 	sock_dequeue_data_all(sock, raw_recv_free);
 	sock->ops = NULL;
@@ -467,20 +462,20 @@ static void nic_op_close(struct socket * sock, __unused message * m)
 		debug_drv_print("no active raw sock at %s", nic->name);
 	}
 
-	sock_reply_close(sock, OK);
+	return OK;
 }
 
-static void nic_ioctl_set_conf(__unused struct socket * sock,
+static int nic_ioctl_set_conf(__unused struct socket * sock,
 				struct nic * nic,
-				message * m)
+				endpoint_t endpt,
+				cp_grant_id_t grant)
 {
 	nwio_ipconf_t ipconf;
 	int err;
 
-	err = copy_from_user(m->m_source, &ipconf, sizeof(ipconf),
-				(cp_grant_id_t) m->IO_GRANT, 0);
+	err = copy_from_user(endpt, &ipconf, sizeof(ipconf), grant, 0);
 	if (err != OK)
-		send_reply(m, err);
+		return err;
 
 	if (ipconf.nwic_flags & NWIC_IPADDR_SET)
 		netif_set_ipaddr(&nic->netif,
@@ -492,51 +487,47 @@ static void nic_ioctl_set_conf(__unused struct socket * sock,
 	if (nic->flags & NWEO_EN_BROAD)
 		nic->netif.flags |= NETIF_FLAG_BROADCAST;
 	
-	send_reply(m, OK);
+
+	return OK;
 }
 
-static void nic_ioctl_get_conf(__unused struct socket * sock,
+static int nic_ioctl_get_conf(__unused struct socket * sock,
 				struct nic * nic,
-				message * m)
+				endpoint_t endpt,
+				cp_grant_id_t grant)
 {
 	nwio_ipconf_t ipconf;
-	int err;
 
 	ipconf.nwic_flags = nic->flags;
 	ipconf.nwic_ipaddr = nic->netif.ip_addr.addr;
 	ipconf.nwic_netmask = nic->netif.netmask.addr;
 	ipconf.nwic_mtu = nic->netif.mtu;
 	
-	err = copy_to_user(m->m_source, &ipconf, sizeof(ipconf),
-				(cp_grant_id_t) m->IO_GRANT, 0);
-	if (err != OK)
-		send_reply(m, err);
-
-	send_reply(m, OK);
+	return copy_to_user(endpt, &ipconf, sizeof(ipconf), grant, 0);
 }
 
-static void nic_ioctl_set_gateway(__unused struct socket * sock,
+static int nic_ioctl_set_gateway(__unused struct socket * sock,
 				struct nic * nic,
-				message * m)
+				endpoint_t endpt,
+				cp_grant_id_t grant)
 {
 	nwio_route_t route;
 	int err;
 
-	err = copy_from_user(m->m_source, &route, sizeof(route),
-				(cp_grant_id_t) m->IO_GRANT, 0);
+	err = copy_from_user(endpt, &route, sizeof(route), grant, 0);
 	if (err != OK)
-		send_reply(m, err);
+		return err;
 
 	netif_set_gw(&nic->netif, (ip_addr_t *)&route.nwr_gateway);
-	
-	send_reply(m, OK);
+
+	return OK;
 }
 
-static void nic_ioctl_get_ethstat(__unused struct socket * sock,
+static int nic_ioctl_get_ethstat(__unused struct socket * sock,
 				struct nic * nic,
-				message * m)
+				endpoint_t endpt,
+				cp_grant_id_t grant)
 {
-	int err;
 	nwio_ethstat_t ethstat;
 
 	debug_drv_print("device /dev/%s", nic->name);
@@ -548,34 +539,27 @@ static void nic_ioctl_get_ethstat(__unused struct socket * sock,
 			!(nic->netif.flags & (NETIF_FLAG_ETHERNET |
 				NETIF_FLAG_ETHARP))) {
 		printf("LWIP no such device FUCK\n");
-		send_reply(m, ENODEV);
-		return;
+		return ENODEV;
 	}
 
 	memset(&ethstat, 0, sizeof(ethstat));
 	memcpy(&ethstat.nwes_addr, nic->netif.hwaddr, 6);
 	
-	err = copy_to_user(m->m_source, &ethstat, sizeof(ethstat),
-				(cp_grant_id_t) m->IO_GRANT, 0);
-	if (err != OK)
-		send_reply(m, err);
-
-	send_reply(m, OK);
+	return copy_to_user(endpt, &ethstat, sizeof(ethstat), grant, 0);
 }
 
-static void nic_ioctl_set_ethopt(struct socket * sock,
+static int nic_ioctl_set_ethopt(struct socket * sock,
 				struct nic * nic,
-				message * m)
+				endpoint_t endpt,
+				cp_grant_id_t grant)
 {
 	int err;
 	nwio_ethopt_t ethopt;
 
 	assert(nic);
 
-	if (!sock) {
-		send_reply(m, EINVAL);
-		return;
-	}
+	if (!sock)
+		return EINVAL;
 
 	debug_drv_print("device /dev/%s", nic->name);
 	/*
@@ -585,83 +569,83 @@ static void nic_ioctl_set_ethopt(struct socket * sock,
 	if (!nic->netif.flags & NETIF_FLAG_UP ||
 			!(nic->netif.flags & (NETIF_FLAG_ETHERNET |
 				NETIF_FLAG_ETHARP))) {
-		send_reply(m, ENODEV);
-		return;
+		return ENODEV;
 	}
 
-	err = copy_from_user(m->m_source, &ethopt, sizeof(ethopt),
-				(cp_grant_id_t) m->IO_GRANT, 0);
+	err = copy_from_user(endpt, &ethopt, sizeof(ethopt), grant, 0);
 	if (err != OK)
-		send_reply(m, err);
+		return err;
 
 	/* we want to get data from this sock */
 	if (ethopt.nweo_flags & NWEO_COPY) {
-		if (nic->raw_socket) {
-			send_reply(m, EBUSY);
-			return;
-		}
+		if (nic->raw_socket)
+			return EBUSY;
 
 		nic->raw_socket = sock;
-		debug_drv_print("active raw sock %d at %s",
+		debug_drv_print("active raw sock %ld at %s",
 				get_sock_num(sock), nic->name);
 	}
 
-	send_reply(m, OK);
+	return OK;
 }
 
-static void nic_do_ioctl(struct socket * sock, struct nic * nic, message * m)
+static int nic_do_ioctl(struct socket * sock, struct nic * nic,
+	struct sock_req * req)
 {
-	debug_print("device /dev/%s req %c %d %d",
+	int r;
+
+	debug_print("device /dev/%s req %c %ld %ld",
 			nic->name,
-			(m->REQUEST >> 8) & 0xff,
-			m->REQUEST & 0xff,
-			(m->REQUEST >> 16) & _IOCPARM_MASK);
+			(unsigned char) (req->req >> 8),
+			req->req & 0xff,
+			(req->req >> 16) & _IOCPARM_MASK);
 	
-	debug_drv_print("socket %d", sock ? get_sock_num(sock) : -1);
+	debug_drv_print("socket %ld", sock ? get_sock_num(sock) : -1);
 
-	switch (m->REQUEST) {
+	switch (req->req) {
 	case NWIOSIPCONF:
-		nic_ioctl_set_conf(sock, nic, m);
+		r = nic_ioctl_set_conf(sock, nic, req->endpt, req->grant);
 		break;
 	case NWIOGIPCONF:
-		nic_ioctl_get_conf(sock, nic, m);
+		r = nic_ioctl_get_conf(sock, nic, req->endpt, req->grant);
 		break;
 	case NWIOSIPOROUTE:
-		nic_ioctl_set_gateway(sock, nic, m);
+		r = nic_ioctl_set_gateway(sock, nic, req->endpt, req->grant);
 		break;
 	case NWIOGETHSTAT:
-		nic_ioctl_get_ethstat(sock, nic, m);
+		r = nic_ioctl_get_ethstat(sock, nic, req->endpt, req->grant);
 		break;
 	case NWIOSETHOPT:
-		nic_ioctl_set_ethopt(sock, nic, m);
+		r = nic_ioctl_set_ethopt(sock, nic, req->endpt, req->grant);
 		break;
 	default:
-		send_reply(m, EBADIOCTL);
-		return;
+		r = ENOTTY;
 	}
+
+	return r;
 }
 
-void nic_default_ioctl(message *m)
+int nic_default_ioctl(struct sock_req *req)
 {
 	struct nic * nic = lookup_nic_default();
 
 	if (nic == NULL) {
 		debug_print("No default nic, reporting error");
-		send_reply(m, EBADIOCTL);
-		return;
+		return ENOTTY;
 	}
 
-	nic_do_ioctl(NULL, nic, m);
+	return nic_do_ioctl(NULL, nic, req);
 }
 
-static void nic_op_ioctl(struct socket * sock, message * m, __unused int blk)
+static int nic_op_ioctl(struct socket * sock, struct sock_req * req,
+	__unused int blk)
 {
-	nic_do_ioctl(sock, (struct nic *)sock->data, m);
+	return nic_do_ioctl(sock, (struct nic *)sock->data, req);
 }
 
-static void nic_op_read(struct socket * sock, message * m, int blk)
+static int nic_op_read(struct socket * sock, struct sock_req * req, int blk)
 {
-	debug_drv_print("sock num %d", get_sock_num(sock));
+	debug_drv_print("sock num %ld", get_sock_num(sock));
 
 	if (sock->recv_head) {
 		/* data available receive immeditely */
@@ -671,59 +655,56 @@ static void nic_op_read(struct socket * sock, message * m, int blk)
 
 		pbuf = sock->recv_head->data;
 
-		ret = raw_receive(m, pbuf);
+		ret = raw_receive(req, pbuf);
 
 		if (ret > 0) {
 			sock_dequeue_data(sock);
 			sock->recv_data_size -= pbuf->tot_len;
 			pbuf_free(pbuf);
 		}
-		sock_reply(sock, ret);
+		return ret;
 	} else if (!blk)
-		send_reply(m, EAGAIN);
+		return EAGAIN;
 	else {
-		/* store the message so we know how to reply */
-		sock->mess = *m;
+		/* store the request so we know how to reply */
+		sock->req = *req;
 		/* operation is being processes */
 		sock->flags |= SOCK_FLG_OP_PENDING;
 
 		debug_print("no data to read, suspending");
+		return EDONTREPLY;
 	}
 }
 
-static void nic_op_write(struct socket * sock, message * m, __unused int blk)
+static int nic_op_write(struct socket * sock, struct sock_req * req,
+	__unused int blk)
 {
 	int ret;
 	struct pbuf * pbuf;
 	struct nic * nic = (struct nic *)sock->data;
 
 	assert(nic);
-	debug_print("device %s data size %d", nic->name,
-			get_sock_num(sock), m->COUNT);
+	debug_print("device %s data size %u", nic->name, req->size);
 
-	pbuf = pbuf_alloc(PBUF_RAW, m->COUNT, PBUF_RAM);
-	if (!pbuf) {
-		ret = ENOMEM;
-		goto write_err;
-	}
+	pbuf = pbuf_alloc(PBUF_RAW, req->size, PBUF_RAM);
+	if (!pbuf)
+		return ENOMEM;
 
-	if ((ret = copy_from_user(m->m_source, pbuf->payload, m->COUNT,
-				(cp_grant_id_t) m->IO_GRANT, 0)) != OK) {
+	if ((ret = copy_from_user(req->endpt, pbuf->payload, req->size,
+			req->grant, 0)) != OK) {
 		pbuf_free(pbuf);
-		goto write_err;
+		return ret;
 	}
 
 	if ((ret = nic->netif.linkoutput(&nic->netif, pbuf) != ERR_OK)) {
 		debug_print("raw linkoutput failed %d", ret);
 		ret = EIO;
 	} else
-		ret = m->COUNT;
-	
+		ret = req->size;
 
 	pbuf_free(pbuf);
-	
-write_err:
-	sock_reply(sock, ret);
+
+	return ret;
 }
 
 static struct sock_ops nic_ops = {
@@ -735,34 +716,28 @@ static struct sock_ops nic_ops = {
 	.select_reply	= generic_op_select_reply
 };
 
-void nic_open(message *m)
+int nic_open(devminor_t minor)
 {
 	struct socket * sock;
 
-	debug_print("device %d", m->DEVICE);
+	debug_print("device %d", minor);
 
-	if (m->DEVICE > MAX_DEVS || devices[m->DEVICE].drv_ep == NONE) {
-		send_reply_open(m, ENODEV);
-		return;
-	}
+	if (minor > MAX_DEVS || devices[minor].drv_ep == NONE)
+		return ENODEV;
 
 	sock = get_unused_sock();
 
-	if (sock == NULL) {
-		send_reply(m, ENODEV);
-		return;
-	}
-	if (sock->ops != NULL) {
-		send_reply(m, EBUSY);
-		return;
-	}
+	if (sock == NULL)
+		return ENODEV;
+	if (sock->ops != NULL)
+		return EBUSY;
 
 	sock->ops = &nic_ops;
 	sock->select_ep = NONE;
 	sock->recv_data_size = 0;
-	sock->data = &devices[m->DEVICE];
+	sock->data = &devices[minor];
 
-	send_reply_open(m, get_sock_num(sock));
+	return get_sock_num(sock);
 }
 
 static int driver_pkt_enqueue(struct packet_q ** head,
diff --git a/servers/lwip/eth.c b/servers/lwip/eth.c
index 9e15bbfb8..74890e5e6 100644
--- a/servers/lwip/eth.c
+++ b/servers/lwip/eth.c
@@ -44,6 +44,7 @@
  */
 
 #include <minix/sysutil.h>
+#include <minix/netsock.h>
 
 #include "lwip/opt.h"
 
diff --git a/servers/lwip/inet_config.c b/servers/lwip/inet_config.c
index b57a70056..d4ec68f10 100644
--- a/servers/lwip/inet_config.c
+++ b/servers/lwip/inet_config.c
@@ -22,11 +22,10 @@ Copyright 1995 Philip Homburg
 #include <minix/type.h>
 #include <minix/sysutil.h>
 #include <minix/syslib.h>
+#include <minix/netsock.h>
 #include "inet_config.h"
 
 #include "proto.h"
-#include <minix/netsock.h>
-
 
 struct eth_conf eth_conf[IP_PORT_MAX];
 struct psip_conf psip_conf[IP_PORT_MAX];
diff --git a/servers/lwip/lwip.c b/servers/lwip/lwip.c
index aec4fc434..8f95b5f42 100644
--- a/servers/lwip/lwip.c
+++ b/servers/lwip/lwip.c
@@ -206,13 +206,13 @@ static void netif_poll_lo(void)
 		netif_poll(netif_lo);
 }
 
-void socket_open(message * m)
+int socket_open(devminor_t minor)
 {
         struct sock_ops * ops;
         struct socket * sock;
         int ret = OK;
 
-        switch (m->DEVICE) {
+        switch (minor) {
         case SOCK_TYPE_TCP:
                 ops = &sock_tcp_ops;
                 break;
@@ -223,21 +223,17 @@ void socket_open(message * m)
                 ops = &sock_raw_ip_ops;
                 break;
         default:
-                if (m->DEVICE - SOCK_TYPES  < MAX_DEVS) {
-                        m->DEVICE -= SOCK_TYPES;
-                        nic_open(m);
-                        return;
-                }
-                printf("LWIP unknown socket type %d\n", m->DEVICE);
-                send_reply_open(m, EINVAL);
-                return;
+                if (minor - SOCK_TYPES  < MAX_DEVS)
+			return nic_open(minor - SOCK_TYPES);
+
+                printf("LWIP unknown socket type %d\n", minor);
+                return EINVAL;
         }
 
         sock = get_unused_sock();
         if (!sock) {
                 printf("LWIP : no free socket\n");
-                send_reply_open(m, EAGAIN);
-                return;
+                return EAGAIN;
         }
 
         sock->ops = ops;
@@ -245,15 +241,16 @@ void socket_open(message * m)
         sock->recv_data_size = 0;
 
         if (sock->ops && sock->ops->open)
-                ret = sock->ops->open(sock, m);
+                ret = sock->ops->open(sock);
 
         if (ret == OK) {
                 debug_print("new socket %ld", get_sock_num(sock));
-                send_reply_open(m, get_sock_num(sock));
+                ret = get_sock_num(sock);
         } else {
                 debug_print("failed %d", ret);
-                send_reply_open(m, ret);
+		/* FIXME: shouldn't sock be freed now? */
         }
+	return ret;
 }
 
 int main(__unused int argc, __unused char ** argv)
diff --git a/servers/lwip/proto.h b/servers/lwip/proto.h
index 400e5b945..ce967a02f 100644
--- a/servers/lwip/proto.h
+++ b/servers/lwip/proto.h
@@ -28,8 +28,8 @@ void nic_init_all(void);
 void driver_request(message * m);
 void driver_up(const char * label, endpoint_t ep);
 /* opens a raw NIC socket */
-void nic_open(message *m);
-void nic_default_ioctl(message *m);
+int nic_open(devminor_t minor);
+int nic_default_ioctl(struct sock_req *req);
 
 /* inet_config.c */
 void inet_read_conf(void);
diff --git a/servers/lwip/raw_ip.c b/servers/lwip/raw_ip.c
index a8b8e5190..d199caac5 100644
--- a/servers/lwip/raw_ip.c
+++ b/servers/lwip/raw_ip.c
@@ -30,7 +30,7 @@ static void raw_ip_recv_free(void * data)
 }
 
 
-static int raw_ip_op_open(struct socket * sock, __unused message * m)
+static int raw_ip_op_open(struct socket * sock)
 {
 	debug_print("socket num %ld", get_sock_num(sock));
 
@@ -56,31 +56,30 @@ static void raw_ip_close(struct socket * sock)
 	sock->ops = NULL;
 }
 
-static void raw_ip_op_close(struct socket * sock, __unused message * m)
+static int raw_ip_op_close(struct socket * sock)
 {
 	debug_print("socket num %ld", get_sock_num(sock));
 
 	raw_ip_close(sock);
 
-	sock_reply_close(sock, OK);
+	return OK;
 }
 
-static int raw_ip_do_receive(message * m,
+static int raw_ip_do_receive(struct sock_req *req,
 			struct pbuf *pbuf)
 {
 	struct pbuf * p;
-	unsigned int rem_len = m->COUNT;
+	size_t rem_len = req->size;
 	unsigned int written = 0, hdr_sz = 0;
 	int err;
 
-	debug_print("user buffer size : %d\n", rem_len);
+	debug_print("user buffer size : %u\n", rem_len);
 
 	for (p = pbuf; p && rem_len; p = p->next) {
 		size_t cp_len;
 
 		cp_len = (rem_len < p->len) ? rem_len : p->len;
-		err = copy_to_user(m->m_source,	p->payload, cp_len,
-				(cp_grant_id_t) m->IO_GRANT,
+		err = copy_to_user(req->endpt, p->payload, cp_len, req->grant,
 				hdr_sz + written);
 
 		if (err != OK)
@@ -108,19 +107,17 @@ static u8_t raw_ip_op_receive(void *arg,
 
 	if (sock->flags & SOCK_FLG_OP_PENDING) {
 		/* we are resuming a suspended operation */
-		ret = raw_ip_do_receive(&sock->mess, pbuf);
+		ret = raw_ip_do_receive(&sock->req, pbuf);
+
+		send_req_reply(&sock->req, ret);
+		sock->flags &= ~SOCK_FLG_OP_PENDING;
 
 		if (ret > 0) {
-			sock_reply(sock, ret);
-			sock->flags &= ~SOCK_FLG_OP_PENDING;
 			if (sock->usr_flags & NWIO_EXCL) {
 				pbuf_free(pbuf);
 				return 1;
 			} else
 				return 0;
-		} else {
-			sock_reply(sock, ret);
-			sock->flags &= ~SOCK_FLG_OP_PENDING;
 		}
 	}
 
@@ -170,14 +167,12 @@ static u8_t raw_ip_op_receive(void *arg,
 	return ret;
 }
 
-static void raw_ip_op_read(struct socket * sock, message * m, int blk)
+static int raw_ip_op_read(struct socket * sock, struct sock_req * req, int blk)
 {
 	debug_print("socket num %ld", get_sock_num(sock));
 
-	if (sock->pcb == NULL) {
-		sock_reply(sock, EIO);
-		return;
-	}
+	if (sock->pcb == NULL)
+		return EIO;
 
 	if (sock->recv_head) {
 		/* data available receive immeditely */
@@ -187,62 +182,57 @@ static void raw_ip_op_read(struct socket * sock, message * m, int blk)
 
 		data = (struct raw_ip_recv_data *) sock->recv_head->data;
 
-		ret = raw_ip_do_receive(m,	data->pbuf);
+		ret = raw_ip_do_receive(req, data->pbuf);
 
 		if (ret > 0) {
 			sock_dequeue_data(sock);
 			sock->recv_data_size -= data->pbuf->tot_len;
 			raw_ip_recv_free(data);
 		}
-		sock_reply(sock, ret);
+		return ret;
 	} else if (!blk)
-		sock_reply(sock, EAGAIN);
+		return EAGAIN;
 	else {
-		/* store the message so we know how to reply */
-		sock->mess = *m;
+		/* store the request so we know how to reply */
+		sock->req = *req;
 		/* operation is being processes */
 		sock->flags |= SOCK_FLG_OP_PENDING;
 
 		debug_print("no data to read, suspending");
+		return EDONTREPLY;
 	}
 }
 
-static void raw_ip_op_write(struct socket * sock, message * m, __unused int blk)
+static int raw_ip_op_write(struct socket * sock, struct sock_req * req,
+	__unused int blk)
 {
 	int ret;
 	struct pbuf * pbuf;
 	struct ip_hdr * ip_hdr;
 
-	debug_print("socket num %ld data size %d",
-			get_sock_num(sock), m->COUNT);
+	debug_print("socket num %ld data size %u",
+			get_sock_num(sock), req->size);
 
-	if (sock->pcb == NULL) {
-		ret = EIO;
-		goto write_err;
-	}
+	if (sock->pcb == NULL)
+		return EIO;
 
-	if ((size_t) m->COUNT > sock->buf_size) {
-		ret = ENOMEM;
-		goto write_err;
-	}
+	if (req->size > sock->buf_size)
+		return ENOMEM;
 
-	pbuf = pbuf_alloc(PBUF_LINK, m->COUNT, PBUF_RAM);
-	if (!pbuf) {
-		ret = ENOMEM;
-		goto write_err;
-	}
+	pbuf = pbuf_alloc(PBUF_LINK, req->size, PBUF_RAM);
+	if (!pbuf)
+		return ENOMEM;
 
-	if ((ret = copy_from_user(m->m_source, pbuf->payload, m->COUNT,
-				(cp_grant_id_t) m->IO_GRANT, 0)) != OK) {
+	if ((ret = copy_from_user(req->endpt, pbuf->payload, req->size,
+				  req->grant, 0)) != OK) {
 		pbuf_free(pbuf);
-		goto write_err;
+		return ret;
 	}
 
 	ip_hdr = (struct ip_hdr *) pbuf->payload;
 	if (pbuf_header(pbuf, -IP_HLEN)) {
 		pbuf_free(pbuf);
-		ret = EIO;
-		goto write_err;
+		return EIO;
 	}
 
 	if ((ret = raw_sendto((struct raw_pcb *)sock->pcb, pbuf,
@@ -250,28 +240,27 @@ static void raw_ip_op_write(struct socket * sock, message * m, __unused int blk)
 		debug_print("raw_sendto failed %d", ret);
 		ret = EIO;
 	} else
-		ret = m->COUNT;
+		ret = req->size;
 	
 
 	pbuf_free(pbuf);
 	
-write_err:
-	sock_reply(sock, ret);
+	return ret;
 }
 
-static void raw_ip_set_opt(struct socket * sock, message * m)
+static int raw_ip_set_opt(struct socket * sock, endpoint_t endpt,
+	cp_grant_id_t grant)
 {
 	int err;
 	nwio_ipopt_t ipopt;
 	struct raw_pcb * pcb;
 
-	err = copy_from_user(m->m_source, &ipopt, sizeof(ipopt),
-				(cp_grant_id_t) m->IO_GRANT, 0);
+	err = copy_from_user(endpt, &ipopt, sizeof(ipopt), grant, 0);
 
 	if (err != OK)
-		sock_reply(sock, err);
+		return err;
 
-	debug_print("ipopt.nwio_flags = 0x%lx", ipopt.nwio_flags);
+	debug_print("ipopt.nwio_flags = 0x%x", ipopt.nwio_flags);
 	debug_print("ipopt.nwio_proto = 0x%x", ipopt.nwio_proto);
 	debug_print("ipopt.nwio_rem = 0x%x",
 				(unsigned int) ipopt.nwio_rem);
@@ -279,8 +268,7 @@ static void raw_ip_set_opt(struct socket * sock, message * m)
 	if (sock->pcb == NULL) {
 		if (!(pcb = raw_new(ipopt.nwio_proto))) {
 			raw_ip_close(sock);
-			sock_reply(sock, ENOMEM);
-			return;
+			return ENOMEM;
 		}
 
 		sock->pcb = pcb;
@@ -289,7 +277,7 @@ static void raw_ip_set_opt(struct socket * sock, message * m)
 
 	if (pcb->protocol != ipopt.nwio_proto) {
 		debug_print("conflicting ip socket protocols\n");
-		sock_reply(sock, EBADIOCTL);
+		return EINVAL;
 	}
 
 	sock->usr_flags = ipopt.nwio_flags;
@@ -297,20 +285,19 @@ static void raw_ip_set_opt(struct socket * sock, message * m)
 #if 0
 	if (raw_bind(pcb, (ip_addr_t *)&ipopt.nwio_rem) == ERR_USE) {
 		raw_ip_close(sock);
-		sock_reply(sock, EADDRINUSE);
-		return;
+		return EADDRINUSE;
 	}
 #endif
 
 	/* register a receive hook */
 	raw_recv((struct raw_pcb *) sock->pcb, raw_ip_op_receive, sock);
 
-	sock_reply(sock, OK);
+	return OK;
 }
 
-static void raw_ip_get_opt(struct socket * sock, message * m)
+static int raw_ip_get_opt(struct socket * sock, endpoint_t endpt,
+	cp_grant_id_t grant)
 {
-	int err;
 	nwio_ipopt_t ipopt;
 	struct raw_pcb * pcb = (struct raw_pcb *) sock->pcb;
 
@@ -319,43 +306,36 @@ static void raw_ip_get_opt(struct socket * sock, message * m)
 	ipopt.nwio_rem = pcb->remote_ip.addr;
 	ipopt.nwio_flags = sock->usr_flags;
 
-	if ((unsigned int) m->COUNT < sizeof(ipopt)) {
-		sock_reply(sock, EINVAL);
-		return;
-	}
-
-	err = copy_to_user(m->m_source, &ipopt, sizeof(ipopt),
-				(cp_grant_id_t) m->IO_GRANT, 0);
-
-	if (err != OK)
-		sock_reply(sock, err);
-
-	sock_reply(sock, OK);
+	return copy_to_user(endpt, &ipopt, sizeof(ipopt), grant, 0);
 }
 
-static void raw_ip_op_ioctl(struct socket * sock, message * m, __unused int blk)
+static int raw_ip_op_ioctl(struct socket * sock, struct sock_req * req,
+	__unused int blk)
 {
-	debug_print("socket num %ld req %c %d %d",
+	int r;
+
+	debug_print("socket num %ld req %c %ld %ld",
 			get_sock_num(sock),
-			(m->REQUEST >> 8) & 0xff,
-			m->REQUEST & 0xff,
-			(m->REQUEST >> 16) & _IOCPARM_MASK);
+			(unsigned char) (req->req >> 8),
+			req->req & 0xff,
+			(req->req >> 16) & _IOCPARM_MASK);
 	
-	switch (m->REQUEST) {
+	switch (req->req) {
 	case NWIOSIPOPT:
-		raw_ip_set_opt(sock, m);
+		r = raw_ip_set_opt(sock, req->endpt, req->grant);
 		break;
 	case NWIOGIPOPT:
-		raw_ip_get_opt(sock, m);
+		r = raw_ip_get_opt(sock, req->endpt, req->grant);
 		break;
 	default:
 		/*
 		 * /dev/ip can be also accessed as a default device to be
 		 * configured
 		 */
-		nic_default_ioctl(m);
-		return;
+		r = nic_default_ioctl(req);
 	}
+
+	return r;
 }
 
 struct sock_ops sock_raw_ip_ops = {
diff --git a/servers/lwip/tcp.c b/servers/lwip/tcp.c
index 3a33e8092..0d636fcee 100644
--- a/servers/lwip/tcp.c
+++ b/servers/lwip/tcp.c
@@ -65,9 +65,10 @@ static void tcp_error_callback(void *arg, err_t err)
 	default:
 		perr = EIO;
 	}
-	
+
+	/* FIXME: what if this is for a write that was already replied to? */
 	if (sock->flags & SOCK_FLG_OP_PENDING) {
-		sock_reply(sock, perr);
+		send_req_reply(&sock->req, perr);
 		sock->flags &= ~SOCK_FLG_OP_PENDING;
 	} else if (sock_select_set(sock))
 		sock_select_notify(sock);
@@ -98,7 +99,7 @@ static int tcp_fill_new_socket(struct socket * sock, struct tcp_pcb * pcb)
 	return OK;
 }
 
-static int tcp_op_open(struct socket * sock, __unused message * m)
+static int tcp_op_open(struct socket * sock)
 {
 	struct tcp_pcb * pcb;
 	int ret;
@@ -142,7 +143,7 @@ static void free_wbuf_chain(struct wbuf_chain * wc)
 	debug_free(wc);
 }
 
-static void tcp_op_close(struct socket * sock, __unused message * m)
+static int tcp_op_close(struct socket * sock)
 {
 	debug_tcp_print("socket num %ld", get_sock_num(sock));
 
@@ -175,11 +176,12 @@ static void tcp_op_close(struct socket * sock, __unused message * m)
 	}
 	debug_tcp_print("freed TX data");
 
-	sock_reply_close(sock, OK);
 	debug_tcp_print("socket unused");
 
 	/* mark it as unused */
 	sock->ops = NULL;
+
+	return OK;
 }
 
 __unused static void print_tcp_payload(unsigned char * buf, int len)
@@ -195,14 +197,14 @@ __unused static void print_tcp_payload(unsigned char * buf, int len)
 	kputc('\n');
 }
 
-static int read_from_tcp(struct socket * sock, message * m)
+static int read_from_tcp(struct socket * sock, struct sock_req * req)
 {
 	unsigned int rem_buf, written = 0;
 	struct pbuf * p;
 
 	assert(!(sock->flags & SOCK_FLG_OP_LISTENING) && sock->recv_head);
 
-	rem_buf = m->COUNT;
+	rem_buf = req->size;
 
 	debug_tcp_print("socket num %ld recv buff sz %d", get_sock_num(sock), rem_buf);
 
@@ -220,8 +222,8 @@ static int read_from_tcp(struct socket * sock, message * m)
 #if 0
 			print_tcp_payload(p->payload, p->len);
 #endif
-			err = copy_to_user(m->m_source, p->payload, p->len,
-					(cp_grant_id_t) m->IO_GRANT, written);
+			err = copy_to_user(req->endpt, p->payload, p->len,
+					req->grant, written);
 			if (err != OK)
 				goto cp_error;
 			sock->recv_data_size -= p->len;
@@ -261,8 +263,8 @@ static int read_from_tcp(struct socket * sock, message * m)
 #if 0
 			print_tcp_payload(p->payload, rem_buf);
 #endif
-			err = copy_to_user(m->m_source, p->payload, rem_buf,
-					(cp_grant_id_t) m->IO_GRANT, written);
+			err = copy_to_user(req->endpt, p->payload, rem_buf,
+					req->grant, written);
 			if (err != OK)
 				goto cp_error;
 			sock->recv_data_size -= rem_buf;
@@ -292,37 +294,36 @@ cp_error:
 		return EFAULT;
 }
 
-static void tcp_op_read(struct socket * sock, message * m, int blk)
+static int tcp_op_read(struct socket * sock, struct sock_req * req, int blk)
 {
 	debug_tcp_print("socket num %ld", get_sock_num(sock));
 
 	if (!sock->pcb || ((struct tcp_pcb *) sock->pcb)->state !=
 							ESTABLISHED) {
 		debug_tcp_print("Connection not established\n");
-		sock_reply(sock, ENOTCONN);
-		return;
+		return ENOTCONN;
 	}
 	if (sock->recv_head) {
 		/* data available receive immeditely */
-		int ret = read_from_tcp(sock,  m);
+		int ret = read_from_tcp(sock, req);
 		debug_tcp_print("read op finished");
-		sock_reply(sock, ret);
+		return ret;
 	} else {
 		if (sock->flags & SOCK_FLG_CLOSED) {
 			printf("socket %ld already closed!!! call from %d\n",
-					get_sock_num(sock), m->USER_ENDPT);
+					get_sock_num(sock), req->endpt);
 			do_tcp_debug = 1;
-			sock_reply(sock, 0);
-			return;
+			return 0;
 		}
                 if (!blk) {
                         debug_tcp_print("reading would block -> EAGAIN");
-                        sock_reply(sock, EAGAIN);
-                        return;
+                        return EAGAIN;
                 }
 		/* operation is being processed */
+		sock->req = *req;
 		debug_tcp_print("no data to read, suspending");
 		sock->flags |= SOCK_FLG_OP_PENDING | SOCK_FLG_OP_READING;
+		return EDONTREPLY;
 	}
 }
 
@@ -389,7 +390,8 @@ static struct wbuf * wbuf_ack_sent(struct socket * sock, unsigned int sz)
 	return wc->head;
 }
 
-static void tcp_op_write(struct socket * sock, message * m, __unused int blk)
+static int tcp_op_write(struct socket * sock, struct sock_req * req,
+	__unused int blk)
 {
 	int ret;
 	struct wbuf * wbuf;
@@ -397,12 +399,10 @@ static void tcp_op_write(struct socket * sock, message * m, __unused int blk)
 	u8_t flgs = 0;
 
 
-	if (!sock->pcb) {
-		sock_reply(sock, ENOTCONN);
-		return;
-	}
+	if (!sock->pcb)
+		return ENOTCONN;
 
-	usr_buf_len = m->COUNT;
+	usr_buf_len = req->size;
 	debug_tcp_print("socket num %ld data size %d",
 			get_sock_num(sock), usr_buf_len);
 
@@ -413,8 +413,7 @@ static void tcp_op_write(struct socket * sock, message * m, __unused int blk)
 	if (sock->buf_size >= TCP_BUF_SIZE) {
 		/* FIXME do not block for now */
 		debug_tcp_print("WARNING : tcp buffers too large, cannot allocate more");
-		sock_reply(sock, ENOMEM);
-		return;
+		return ENOMEM;
 	}
 	/*
 	 * Never let the allocated buffers grow more than to 2xTCP_BUF_SIZE and
@@ -426,13 +425,12 @@ static void tcp_op_write(struct socket * sock, message * m, __unused int blk)
 	
 	if (!wbuf) {
 		debug_tcp_print("cannot allocate new buffer of %d bytes", usr_buf_len);
-		sock_reply(sock, ENOMEM);
+		return ENOMEM;
 	}
 
-	if ((ret = copy_from_user(m->m_source, wbuf->data, usr_buf_len,
-				(cp_grant_id_t) m->IO_GRANT, 0)) != OK) {
-		sock_reply(sock, ret);
-		return;
+	if ((ret = copy_from_user(req->endpt, wbuf->data, usr_buf_len,
+				req->grant, 0)) != OK) {
+		return ret;
 	}
 
 	wbuf->written = 0;
@@ -453,15 +451,18 @@ static void tcp_op_write(struct socket * sock, message * m, __unused int blk)
 			debug_tcp_print("unsent %p remains %d\n", wbuf, wbuf->rem_len);
 		}
 		debug_tcp_print("returns %d\n", usr_buf_len);
-		sock_reply(sock, usr_buf_len);
 		/*
 		 * We cannot accept new operations (write). We set the flag
 		 * after sending reply not to revive only. We could deadlock.
 		 */
+		/*
+		 * FIXME: this looks like bad logic. We acknowledge the write
+		 * operation, so we will never reply to it or cancel it later.
+		 */
 		if (sock->buf_size >= TCP_BUF_SIZE)
 			sock->flags |= SOCK_FLG_OP_PENDING;
 
-		return;
+		return usr_buf_len;
 	}
 
 	/*
@@ -496,15 +497,20 @@ static void tcp_op_write(struct socket * sock, message * m, __unused int blk)
 		 * in this case, we are going to reply immediatly
 		 */
 		debug_tcp_print("returns %d\n", usr_buf_len);
-		sock_reply(sock, usr_buf_len);
 		sock->flags |= SOCK_FLG_OP_WRITING;
+		/*
+		 * FIXME: this looks like bad logic. We acknowledge the write
+		 * operation, so we will never reply to it or cancel it later.
+		 */
 		if (sock->buf_size >= TCP_BUF_SIZE)
 			sock->flags |= SOCK_FLG_OP_PENDING;
+		return usr_buf_len;
 	} else
-		sock_reply(sock, EIO);
+		return EIO;
 }
 
-static void tcp_set_conf(struct socket * sock, message * m)
+static int tcp_set_conf(struct socket * sock, endpoint_t endpt,
+	cp_grant_id_t grant)
 {
 	int err;
 	nwio_tcpconf_t tconf;
@@ -514,13 +520,12 @@ static void tcp_set_conf(struct socket * sock, message * m)
 
 	assert(pcb);
 
-	err = copy_from_user(m->m_source, &tconf, sizeof(tconf),
-				(cp_grant_id_t) m->IO_GRANT, 0);
+	err = copy_from_user(endpt, &tconf, sizeof(tconf), grant, 0);
 
 	if (err != OK)
-		sock_reply(sock, err);
+		return err;
 
-	debug_tcp_print("tconf.nwtc_flags = 0x%lx", tconf.nwtc_flags);
+	debug_tcp_print("tconf.nwtc_flags = 0x%x", tconf.nwtc_flags);
 	debug_tcp_print("tconf.nwtc_remaddr = 0x%x",
 				(unsigned int) tconf.nwtc_remaddr);
 	debug_tcp_print("tconf.nwtc_remport = 0x%x", ntohs(tconf.nwtc_remport));
@@ -538,17 +543,16 @@ static void tcp_set_conf(struct socket * sock, message * m)
 	if (sock->usr_flags & NWTC_LP_SET) {
 		/* FIXME the user library can only bind to ANY anyway */
 		if (tcp_bind(pcb, IP_ADDR_ANY, ntohs(tconf.nwtc_locport)) == ERR_USE) {
-			sock_reply(sock, EADDRINUSE);
-			return;
+			return EADDRINUSE;
 		}
 	}
-	
-	sock_reply(sock, OK);
+
+	return OK;
 }
 
-static void tcp_get_conf(struct socket * sock, message * m)
+static int tcp_get_conf(struct socket * sock, endpoint_t endpt,
+	cp_grant_id_t grant)
 {
-	int err;
 	nwio_tcpconf_t tconf;
 	struct tcp_pcb * pcb = (struct tcp_pcb *) sock->pcb;
 
@@ -562,7 +566,7 @@ static void tcp_get_conf(struct socket * sock, message * m)
 	tconf.nwtc_remport = htons(pcb->remote_port);
 	tconf.nwtc_flags = sock->usr_flags;
 
-	debug_tcp_print("tconf.nwtc_flags = 0x%lx", tconf.nwtc_flags);
+	debug_tcp_print("tconf.nwtc_flags = 0x%x", tconf.nwtc_flags);
 	debug_tcp_print("tconf.nwtc_remaddr = 0x%x",
 				(unsigned int) tconf.nwtc_remaddr);
 	debug_tcp_print("tconf.nwtc_remport = 0x%x", ntohs(tconf.nwtc_remport));
@@ -570,18 +574,7 @@ static void tcp_get_conf(struct socket * sock, message * m)
 				(unsigned int) tconf.nwtc_locaddr);
 	debug_tcp_print("tconf.nwtc_locport = 0x%x", ntohs(tconf.nwtc_locport));
 
-	if ((unsigned int) m->COUNT < sizeof(tconf)) {
-		sock_reply(sock, EINVAL);
-		return;
-	}
-	
-	err = copy_to_user(m->m_source, &tconf, sizeof(tconf),
-				(cp_grant_id_t) m->IO_GRANT, 0);
-
-	if (err != OK)
-		sock_reply(sock, err);
-
-	sock_reply(sock, OK);
+	return copy_to_user(endpt, &tconf, sizeof(tconf), grant, 0);
 }
 
 static int enqueue_rcv_data(struct socket * sock, struct pbuf * pbuf)
@@ -627,7 +620,7 @@ static err_t tcp_recv_callback(void *arg,
 		/* wake up the reader and report EOF */
 		if (sock->flags & SOCK_FLG_OP_PENDING &&
 				sock->flags & SOCK_FLG_OP_READING) {
-			sock_reply(sock, 0);
+			send_req_reply(&sock->req, 0);
 			sock->flags &= ~(SOCK_FLG_OP_PENDING |
 					SOCK_FLG_OP_READING);
 		}
@@ -655,9 +648,9 @@ static err_t tcp_recv_callback(void *arg,
 	 */
 	if (sock->flags & SOCK_FLG_OP_PENDING) {
 		if (sock->flags & SOCK_FLG_OP_READING) {
-			ret = read_from_tcp(sock, &sock->mess);
+			ret = read_from_tcp(sock, &sock->req);
 			debug_tcp_print("read op finished");
-			sock_reply(sock, ret);
+			send_req_reply(&sock->req, ret);
 			sock->flags &= ~(SOCK_FLG_OP_PENDING |
 					SOCK_FLG_OP_READING);
 		}
@@ -692,7 +685,11 @@ static err_t tcp_sent_callback(void *arg, struct tcp_pcb *tpcb, u16_t len)
 	assert((struct tcp_pcb *)sock->pcb == tpcb);
 
 	/* operation must have been canceled, do not send any other data */
-	if (!sock->flags & SOCK_FLG_OP_PENDING)
+	/*
+	 * FIXME: this looks like bad logic. We already acknowledged the write
+	 * operation, so we should not set or check the OP_PENDING flag..
+	 */
+	if (!(sock->flags & SOCK_FLG_OP_PENDING))
 		return ERR_OK;
 
 	wbuf = wbuf_ack_sent(sock, len);
@@ -797,7 +794,7 @@ static err_t tcp_connected_callback(void *arg,
 
 	tcp_sent(tpcb, tcp_sent_callback);
 	tcp_recv(tpcb, tcp_recv_callback);
-	sock_reply(sock, OK);
+	send_req_reply(&sock->req, OK);
 	sock->flags &= ~(SOCK_FLG_OP_PENDING | SOCK_FLG_OP_CONNECTING);
 
 	/* revive does the sock_select_notify() for us */
@@ -805,7 +802,7 @@ static err_t tcp_connected_callback(void *arg,
 	return ERR_OK;
 }
 
-static void tcp_op_connect(struct socket * sock)
+static int tcp_op_connect(struct socket * sock, struct sock_req * req)
 {
 	ip_addr_t remaddr;
 	struct tcp_pcb * pcb;
@@ -821,16 +818,18 @@ static void tcp_op_connect(struct socket * sock)
 	/* try to connect now */
 	pcb = (struct tcp_pcb *) sock->pcb;
 	remaddr = pcb->remote_ip;
+	sock->req = *req;
 	err = tcp_connect(pcb, &remaddr, pcb->remote_port,
 				tcp_connected_callback);
 	if (err == ERR_VAL)
 		panic("Wrong tcp_connect arguments");
 	if (err != ERR_OK)
 		panic("Other tcp_connect error %d\n", err);
+	return EDONTREPLY;
 }
 
 static int tcp_do_accept(struct socket * listen_sock,
-			message * m,
+			struct sock_req * req,
 			struct tcp_pcb * newpcb)
 {
 	struct socket * newsock;
@@ -839,8 +838,8 @@ static int tcp_do_accept(struct socket * listen_sock,
 
 	debug_tcp_print("socket num %ld", get_sock_num(listen_sock));
 
-	if ((ret = copy_from_user(m->m_source, &sock_num, sizeof(sock_num),
-				(cp_grant_id_t) m->IO_GRANT, 0)) != OK)
+	if ((ret = copy_from_user(req->endpt, &sock_num, sizeof(sock_num),
+				req->grant, 0)) != OK)
 		return EFAULT;
 	if (!is_valid_sock_num(sock_num))
 		return EBADF;
@@ -877,8 +876,8 @@ static err_t tcp_accept_callback(void *arg, struct tcp_pcb *newpcb, err_t err)
 	if (sock->flags & SOCK_FLG_OP_PENDING) {
 		int ret;
 
-		ret = tcp_do_accept(sock, &sock->mess, newpcb);
-		sock_reply(sock, ret);
+		ret = tcp_do_accept(sock, &sock->req, newpcb);
+		send_req_reply(&sock->req, ret);
 		sock->flags &= ~SOCK_FLG_OP_PENDING;
 		if (ret == OK) {
 			return ERR_OK;
@@ -900,15 +899,18 @@ static err_t tcp_accept_callback(void *arg, struct tcp_pcb *newpcb, err_t err)
 	return ERR_OK;
 }
 
-static void tcp_op_listen(struct socket * sock, message * m)
+static int tcp_op_listen(struct socket * sock, endpoint_t endpt,
+	cp_grant_id_t grant)
 {
 	int backlog, err;
 	struct tcp_pcb * new_pcb;
 
 	debug_tcp_print("socket num %ld", get_sock_num(sock));
 
-	err = copy_from_user(m->m_source, &backlog, sizeof(backlog),
-				(cp_grant_id_t) m->IO_GRANT, 0);
+	err = copy_from_user(endpt, &backlog, sizeof(backlog), grant, 0);
+
+	if (err != OK)
+		return err;
 
 	new_pcb = tcp_listen_with_backlog((struct tcp_pcb *) sock->pcb,
 							(u8_t) backlog);
@@ -916,8 +918,7 @@ static void tcp_op_listen(struct socket * sock, message * m)
 
 	if (!new_pcb) {
 		debug_tcp_print("Cannot listen on socket %ld", get_sock_num(sock));
-		sock_reply(sock, EGENERIC);
-		return;
+		return EIO;
 	}
 
 	/* advertise that this socket is willing to accept connections */
@@ -925,17 +926,16 @@ static void tcp_op_listen(struct socket * sock, message * m)
 	sock->flags |= SOCK_FLG_OP_LISTENING;
 
 	sock->pcb = new_pcb;
-	sock_reply(sock, OK);
+	return OK;
 }
 
-static void tcp_op_accept(struct socket * sock, message * m)
+static int tcp_op_accept(struct socket * sock, struct sock_req * req)
 {
 	debug_tcp_print("socket num %ld", get_sock_num(sock));
 
 	if (!(sock->flags & SOCK_FLG_OP_LISTENING)) {
 		debug_tcp_print("socket %ld does not listen\n", get_sock_num(sock));
-		sock_reply(sock, EINVAL);
-		return;
+		return EINVAL;
 	}
 
 	/* there is a connection ready to be accepted */
@@ -946,19 +946,22 @@ static void tcp_op_accept(struct socket * sock, message * m)
 		pcb = (struct tcp_pcb *) sock->recv_head->data;
 		assert(pcb);
 
-		ret = tcp_do_accept(sock, m, pcb);
-		sock_reply(sock, ret);
+		ret = tcp_do_accept(sock, req, pcb);
 		if (ret == OK)
 			sock_dequeue_data(sock);
-		return;
+		return ret;
 	}
 
 	debug_tcp_print("no ready connection, suspending\n");
 
+	sock->req = *req;
+
 	sock->flags |= SOCK_FLG_OP_PENDING;
+
+	return EDONTREPLY;
 }
 
-static void tcp_op_shutdown_tx(struct socket * sock)
+static int tcp_op_shutdown_tx(struct socket * sock)
 {
 	err_t err;
 
@@ -968,17 +971,16 @@ static void tcp_op_shutdown_tx(struct socket * sock)
 
 	switch (err) {
 	case ERR_OK:
-		sock_reply(sock, OK);
-		break;
+		return OK;
 	case ERR_CONN:
-		sock_reply(sock, ENOTCONN);
-		break;
+		return ENOTCONN;
 	default:
-		sock_reply(sock, EGENERIC);
+		return EIO;
 	}
 }
 
-static void tcp_op_get_cookie(struct socket * sock, message * m)
+static int tcp_op_get_cookie(struct socket * sock, endpoint_t endpt,
+	cp_grant_id_t grant)
 {
 	tcp_cookie_t cookie;
 	unsigned int sock_num;
@@ -988,16 +990,12 @@ static void tcp_op_get_cookie(struct socket * sock, message * m)
 	sock_num = get_sock_num(sock);
 	memcpy(&cookie, &sock_num, sizeof(sock_num));
 
-	if (copy_to_user(m->m_source, &cookie, sizeof(sock),
-			(cp_grant_id_t) m->IO_GRANT, 0) == OK)
-		sock_reply(sock, OK);
-	else
-		sock_reply(sock, EFAULT);
+	return copy_to_user(endpt, &cookie, sizeof(sock), grant, 0);
 }
 
-static void tcp_get_opt(struct socket * sock, message * m)
+static int tcp_get_opt(struct socket * sock, endpoint_t endpt,
+	cp_grant_id_t grant)
 {
-	int err;
 	nwio_tcpopt_t tcpopt;
 	struct tcp_pcb * pcb = (struct tcp_pcb *) sock->pcb;
 
@@ -1005,26 +1003,15 @@ static void tcp_get_opt(struct socket * sock, message * m)
 
 	assert(pcb);
 
-	if ((unsigned int) m->COUNT < sizeof(tcpopt)) {
-		sock_reply(sock, EINVAL);
-		return;
-	}
-
 	/* FIXME : not used by the userspace library */
 	tcpopt.nwto_flags = 0;
 	
-	err = copy_to_user(m->m_source, &tcpopt, sizeof(tcpopt),
-				(cp_grant_id_t) m->IO_GRANT, 0);
-
-	if (err != OK)
-		sock_reply(sock, err);
-
-	sock_reply(sock, OK);
+	return copy_to_user(endpt, &tcpopt, sizeof(tcpopt), grant, 0);
 }
 
-static void tcp_set_opt(struct socket * sock, message * m)
+static int tcp_set_opt(struct socket * sock, endpoint_t endpt,
+	cp_grant_id_t grant)
 {
-	int err;
 	nwio_tcpopt_t tcpopt;
 	struct tcp_pcb * pcb = (struct tcp_pcb *) sock->pcb;
 
@@ -1032,69 +1019,64 @@ static void tcp_set_opt(struct socket * sock, message * m)
 
 	assert(pcb);
 
-	err = copy_from_user(m->m_source, &tcpopt, sizeof(tcpopt),
-				(cp_grant_id_t) m->IO_GRANT, 0);
-
-	if (err != OK)
-		sock_reply(sock, err);
-
 	/* FIXME : The userspace library does not use this */
 
-	sock_reply(sock, OK);
+	return copy_from_user(endpt, &tcpopt, sizeof(tcpopt), grant, 0);
 }
 
-static void tcp_op_ioctl(struct socket * sock, message * m, __unused int blk)
+static int tcp_op_ioctl(struct socket * sock, struct sock_req * req,
+	__unused int blk)
 {
-	if (!sock->pcb) {
-		sock_reply(sock, ENOTCONN);
-		return;
-	}
+	int r;
 
-	debug_tcp_print("socket num %ld req %c %d %d",
+	if (!sock->pcb)
+		return ENOTCONN;
+
+	debug_tcp_print("socket num %ld req %c %ld %ld",
 			get_sock_num(sock),
-			(m->REQUEST >> 8) & 0xff,
-			m->REQUEST & 0xff,
-			(m->REQUEST >> 16) & _IOCPARM_MASK);
+			(unsigned char) (req->req >> 8),
+			req->req & 0xff,
+			(req->req >> 16) & _IOCPARM_MASK);
 	
-	switch (m->REQUEST) {
+	switch (req->req) {
 	case NWIOGTCPCONF:
-		tcp_get_conf(sock, m);
+		r = tcp_get_conf(sock, req->endpt, req->grant);
 		break;
 	case NWIOSTCPCONF:
-		tcp_set_conf(sock, m);
+		r = tcp_set_conf(sock, req->endpt, req->grant);
 		break;
 	case NWIOTCPCONN:
-		tcp_op_connect(sock);
+		r = tcp_op_connect(sock, req);
 		break;
 	case NWIOTCPLISTENQ:
-		tcp_op_listen(sock, m);
+		r = tcp_op_listen(sock, req->endpt, req->grant);
 		break;
 	case NWIOGTCPCOOKIE:
-		tcp_op_get_cookie(sock, m);
+		r = tcp_op_get_cookie(sock, req->endpt, req->grant);
 		break;
 	case NWIOTCPACCEPTTO:
-		tcp_op_accept(sock, m);
+		r = tcp_op_accept(sock, req);
 		break;
 	case NWIOTCPSHUTDOWN:
-		tcp_op_shutdown_tx(sock);
+		r = tcp_op_shutdown_tx(sock);
 		break;
 	case NWIOGTCPOPT:
-		tcp_get_opt(sock, m);
+		r = tcp_get_opt(sock, req->endpt, req->grant);
 		break;
 	case NWIOSTCPOPT:
-		tcp_set_opt(sock, m);
+		r = tcp_set_opt(sock, req->endpt, req->grant);
 		break;
 	default:
-		sock_reply(sock, EBADIOCTL);
-		return;
+		r = ENOTTY;
 	}
+
+	return r;
 }
 
-static void tcp_op_select(struct socket * sock, __unused message * m)
+static int tcp_op_select(struct socket * sock, unsigned int sel)
 {
-	int retsel = 0, sel;
+	int retsel = 0;
 
-	sel = m->USER_ENDPT;
 	debug_tcp_print("socket num %ld 0x%x", get_sock_num(sock), sel);
 	
 	/* in this case any operation would block, no error */
@@ -1112,8 +1094,7 @@ static void tcp_op_select(struct socket * sock, __unused message * m)
 			if (sel & SEL_ERR)
 				sock->flags |= SOCK_FLG_SEL_ERROR;
 		}
-		sock_reply_select(sock, 0);
-		return;
+		return 0;
 	}
 
 	if (sel & SEL_RD) {
@@ -1158,18 +1139,19 @@ static void tcp_op_select(struct socket * sock, __unused message * m)
 	if (sel & SEL_ERR && sel & SEL_NOTIFY)
 		sock->flags |= SOCK_FLG_SEL_ERROR;
 
-	sock_reply_select(sock, retsel);
+	return retsel;
 }
 
-static void tcp_op_select_reply(struct socket * sock, message * m)
+static int tcp_op_select_reply(struct socket * sock)
 {
+	unsigned int sel = 0;
+
 	assert(sock->select_ep != NONE);
 	debug_tcp_print("socket num %ld", get_sock_num(sock));
 
-
 	if (sock->flags & SOCK_FLG_OP_PENDING) {
 		debug_tcp_print("WARNING socket still blocking!");
-		return;
+		return EDONTREPLY;
 	}
 
 	if (sock->flags & SOCK_FLG_SEL_READ) {
@@ -1178,7 +1160,7 @@ static void tcp_op_select_reply(struct socket * sock, message * m)
 			 (!(sock->flags & SOCK_FLG_OP_LISTENING) && 
 			 ((struct tcp_pcb *) sock->pcb)->state !=
 			 ESTABLISHED)) {
-			m->DEV_SEL_OPS |= SEL_RD;
+			sel |= SEL_RD;
 			debug_tcp_print("read won't block");
 		}
 	}
@@ -1187,13 +1169,15 @@ static void tcp_op_select_reply(struct socket * sock, message * m)
 			(sock->pcb == NULL ||
 			 ((struct tcp_pcb *) sock->pcb)->state ==
 			 ESTABLISHED)) {
-		m->DEV_SEL_OPS |= SEL_WR;
+		sel |= SEL_WR;
 		debug_tcp_print("write won't block");
 	}
 
-	if (m->DEV_SEL_OPS) 
+	if (sel)
 		sock->flags &= ~(SOCK_FLG_SEL_WRITE | SOCK_FLG_SEL_READ |
 							SOCK_FLG_SEL_ERROR);
+
+	return sel;
 }
 
 struct sock_ops sock_tcp_ops = {
diff --git a/servers/lwip/udp.c b/servers/lwip/udp.c
index c3367c01a..97326a21b 100644
--- a/servers/lwip/udp.c
+++ b/servers/lwip/udp.c
@@ -41,7 +41,7 @@ static void udp_recv_free(void * data)
 	debug_free(data);
 }
 
-static int udp_op_open(struct socket * sock, __unused message * m)
+static int udp_op_open(struct socket * sock)
 {
 	struct udp_pcb * pcb;
 
@@ -58,7 +58,7 @@ static int udp_op_open(struct socket * sock, __unused message * m)
 	return OK;
 }
 
-static void udp_op_close(struct socket * sock, __unused message * m)
+static int udp_op_close(struct socket * sock)
 {
 	debug_udp_print("socket num %ld", get_sock_num(sock));
 
@@ -72,22 +72,22 @@ static void udp_op_close(struct socket * sock, __unused message * m)
 	/* mark it as unused */
 	sock->ops = NULL;
 
-	sock_reply_close(sock, OK);
+	return OK;
 }
 
 static int udp_do_receive(struct socket * sock,
-			message * m,
+			struct sock_req * req,
 			struct udp_pcb *pcb,
 			struct pbuf *pbuf,
 			ip_addr_t *addr,
 			u16_t port)
 {
 	struct pbuf * p;
-	unsigned int rem_len = m->COUNT;
+	size_t rem_len = req->size;
 	unsigned int written = 0, hdr_sz = 0;
 	int err;
 
-	debug_udp_print("user buffer size : %d", rem_len);
+	debug_udp_print("user buffer size : %u", rem_len);
 
 	/* FIXME make it both a single copy */
 	if (!(sock->usr_flags & NWUO_RWDATONLY)) {
@@ -101,10 +101,8 @@ static int udp_do_receive(struct socket * sock,
 		hdr.uih_data_len = 0;
 		hdr.uih_ip_opt_len = 0;
 
-		err = copy_to_user(m->m_source,
-				&hdr, sizeof(hdr),
-				(cp_grant_id_t) m->IO_GRANT,
-				0);
+		err = copy_to_user(req->endpt, &hdr, sizeof(hdr), req->grant,
+			0);
 
 		if (err != OK)
 			return err;
@@ -116,8 +114,7 @@ static int udp_do_receive(struct socket * sock,
 		size_t cp_len;
 
 		cp_len = (rem_len < p->len) ? rem_len : p->len;
-		err = copy_to_user(m->m_source,	p->payload, cp_len,
-				(cp_grant_id_t) m->IO_GRANT,
+		err = copy_to_user(req->endpt, p->payload, cp_len, req->grant,
 				hdr_sz + written);
 
 		if (err != OK)
@@ -147,16 +144,14 @@ static void udp_recv_callback(void *arg,
 		/* we are resuming a suspended operation */
 		int ret;
 
-		ret = udp_do_receive(sock, &sock->mess, pcb, pbuf, addr, port);
+		ret = udp_do_receive(sock, &sock->req, pcb, pbuf, addr, port);
+
+		send_req_reply(&sock->req, ret);
+		sock->flags &= ~SOCK_FLG_OP_PENDING;
 
 		if (ret > 0) {
 			pbuf_free(pbuf);
-			sock_reply(sock, ret);
-			sock->flags &= ~SOCK_FLG_OP_PENDING;
 			return;
-		} else {
-			sock_reply(sock, ret);
-			sock->flags &= ~SOCK_FLG_OP_PENDING;
 		}
 	}
 
@@ -193,7 +188,7 @@ static void udp_recv_callback(void *arg,
 		sock_select_notify(sock);
 }
 
-static void udp_op_read(struct socket * sock, message * m, int blk)
+static int udp_op_read(struct socket * sock, struct sock_req * req, int blk)
 {
 	debug_udp_print("socket num %ld", get_sock_num(sock));
 
@@ -205,7 +200,7 @@ static void udp_op_read(struct socket * sock, message * m, int blk)
 
 		data = (struct udp_recv_data *) sock->recv_head->data;
 
-		ret = udp_do_receive(sock, m, (struct udp_pcb *) sock->pcb,
+		ret = udp_do_receive(sock, req, (struct udp_pcb *) sock->pcb,
 					data->pbuf, &data->ip, data->port);
 
 		if (ret > 0) {
@@ -213,36 +208,37 @@ static void udp_op_read(struct socket * sock, message * m, int blk)
 			sock->recv_data_size -= data->pbuf->tot_len;
 			udp_recv_free(data);
 		}
-		sock_reply(sock, ret);
+		return ret;
 	} else if (!blk)
-		sock_reply(sock, EAGAIN);
+		return EAGAIN;
 	else {
 		/* store the message so we know how to reply */
-		sock->mess = *m;
+		sock->req = *req;
 		/* operation is being processes */
 		sock->flags |= SOCK_FLG_OP_PENDING;
 
 		debug_udp_print("no data to read, suspending\n");
+		return EDONTREPLY;
 	}
 }
 
 static int udp_op_send(struct socket * sock,
 			struct pbuf * pbuf,
-			message * m)
+			size_t size)
 {
 	int err;
 
 	debug_udp_print("pbuf len %d\n", pbuf->len);
 
 	if ((err = udp_send(sock->pcb, pbuf)) == ERR_OK)
-		return m->COUNT;
+		return size;
 	else {
 		debug_udp_print("udp_send failed %d", err);
 		return EIO;
 	}
 }
 
-static int udp_op_sendto(struct socket * sock, struct pbuf * pbuf, message * m)
+static int udp_op_sendto(struct socket * sock, struct pbuf * pbuf, size_t size)
 {
 	int err;
 	udp_io_hdr_t hdr;
@@ -256,47 +252,46 @@ static int udp_op_sendto(struct socket * sock, struct pbuf * pbuf, message * m)
 
 	if ((err = udp_sendto(sock->pcb, pbuf, (ip_addr_t *) &hdr.uih_dst_addr,
 						ntohs(hdr.uih_dst_port))) == ERR_OK)
-		return m->COUNT;
+		return size;
 	else {
 		debug_udp_print("udp_sendto failed %d", err);
 		return EIO;
 	}
 }
 
-static void udp_op_write(struct socket * sock, message * m, __unused int blk)
+static int udp_op_write(struct socket * sock, struct sock_req * req,
+	__unused int blk)
 {
 	int ret;
 	struct pbuf * pbuf;
 
-	debug_udp_print("socket num %ld data size %d",
-			get_sock_num(sock), m->COUNT);
+	debug_udp_print("socket num %ld data size %u",
+			get_sock_num(sock), req->size);
 
-	pbuf = pbuf_alloc(PBUF_TRANSPORT, m->COUNT, PBUF_POOL);
-	if (!pbuf) {
-		ret = ENOMEM;
-		goto write_err;
-	}
+	pbuf = pbuf_alloc(PBUF_TRANSPORT, req->size, PBUF_POOL);
+	if (!pbuf)
+		return ENOMEM;
 
-	if ((ret = copy_from_user(m->m_source, pbuf->payload, m->COUNT,
-				(cp_grant_id_t) m->IO_GRANT, 0)) != OK) {
+	if ((ret = copy_from_user(req->endpt, pbuf->payload, req->size,
+				req->grant, 0)) != OK) {
 		pbuf_free(pbuf);
-		goto write_err;
+		return ret;
 	}
 
 	if (sock->usr_flags & NWUO_RWDATONLY)
-		ret = udp_op_send(sock, pbuf, m);
+		ret = udp_op_send(sock, pbuf, req->size);
 	else
-		ret = udp_op_sendto(sock, pbuf, m);
+		ret = udp_op_sendto(sock, pbuf, req->size);
 
 	if (pbuf_free(pbuf) == 0) {
 		panic("We cannot buffer udp packets yet!");
 	}
-	
-write_err:
-	sock_reply(sock, ret);
+
+	return ret;
 }
 
-static void udp_set_opt(struct socket * sock, message * m)
+static int udp_set_opt(struct socket * sock, endpoint_t endpt,
+	cp_grant_id_t grant)
 {
 	int err;
 	nwio_udpopt_t udpopt;
@@ -305,11 +300,10 @@ static void udp_set_opt(struct socket * sock, message * m)
 
 	assert(pcb);
 
-	err = copy_from_user(m->m_source, &udpopt, sizeof(udpopt),
-				(cp_grant_id_t) m->IO_GRANT, 0);
+	err = copy_from_user(endpt, &udpopt, sizeof(udpopt), grant, 0);
 
 	if (err != OK)
-		sock_reply(sock, err);
+		return err;
 
 	debug_udp_print("udpopt.nwuo_flags = 0x%lx", udpopt.nwuo_flags);
 	debug_udp_print("udpopt.nwuo_remaddr = 0x%x",
@@ -341,16 +335,15 @@ static void udp_set_opt(struct socket * sock, message * m)
 	if (sock->usr_flags & NWUO_LP_SEL)
 		udp_bind(pcb, &loc_ip, 0);
 
-	
 	/* register a receive hook */
 	udp_recv((struct udp_pcb *) sock->pcb, udp_recv_callback, sock);
 
-	sock_reply(sock, OK);
+	return OK;
 }
 
-static void udp_get_opt(struct socket * sock, message * m)
+static int udp_get_opt(struct socket * sock, endpoint_t endpt,
+	cp_grant_id_t grant)
 {
-	int err;
 	nwio_udpopt_t udpopt;
 	struct udp_pcb * pcb = (struct udp_pcb *) sock->pcb;
 
@@ -372,39 +365,32 @@ static void udp_get_opt(struct socket * sock, message * m)
 	debug_udp_print("udpopt.nwuo_locport = 0x%x",
 				ntohs(udpopt.nwuo_locport));
 
-	if ((unsigned int) m->COUNT < sizeof(udpopt)) {
-		sock_reply(sock, EINVAL);
-		return;
-	}
-
-	err = copy_to_user(m->m_source, &udpopt, sizeof(udpopt),
-				(cp_grant_id_t) m->IO_GRANT, 0);
-
-	if (err != OK)
-		sock_reply(sock, err);
-
-	sock_reply(sock, OK);
+	return copy_to_user(endpt, &udpopt, sizeof(udpopt), grant, 0);
 }
 
-static void udp_op_ioctl(struct socket * sock, message * m, __unused int blk)
+static int udp_op_ioctl(struct socket * sock, struct sock_req * req,
+	__unused int blk)
 {
-	debug_udp_print("socket num %ld req %c %d %d",
+	int r;
+
+	debug_udp_print("socket num %ld req %c %ld %ld",
 			get_sock_num(sock),
-			(m->REQUEST >> 8) & 0xff,
-			m->REQUEST & 0xff,
-			(m->REQUEST >> 16) & _IOCPARM_MASK);
+			(unsigned char) (req->req >> 8),
+			req->req & 0xff,
+			(req->req >> 16) & _IOCPARM_MASK);
 
-	switch (m->REQUEST) {
+	switch (req->req) {
 	case NWIOSUDPOPT:
-		udp_set_opt(sock, m);
+		r = udp_set_opt(sock, req->endpt, req->grant);
 		break;
 	case NWIOGUDPOPT:
-		udp_get_opt(sock, m);
+		r = udp_get_opt(sock, req->endpt, req->grant);
 		break;
 	default:
-		sock_reply(sock, EBADIOCTL);
-		return;
+		r = ENOTTY;
 	}
+
+	return r;
 }
 
 struct sock_ops sock_udp_ops = {
-- 
2.44.0