RDMA Read and Write with IB Verbs
http://www.hpcadvisorycouncil.com/pdf/rdma-read-and-write-with-ib-verbs.pdf
Building an RDMA-Capable Application with IB Verbs
http://www.hpcadvisorycouncil.com/pdf/building-an-rdma-capable-application-with-ib-verbs.pdf
mellanox ethernet mode
============================
nvmet_rdma 24576 0 nvmet 24576 1 nvmet_rdma rdma_cm 45056 1 nvmet_rdma ib_cm 45056 1 rdma_cm ib_umad 24576 0 mlx4_ib 147456 0 mlx4_en 110592 0 ib_sa 36864 3 rdma_cm,ib_cm,mlx4_ib ib_mad 49152 4 ib_cm,ib_sa,mlx4_ib,ib_umad ib_core 114688 8 rdma_cm,ib_cm,ib_sa,iw_cm,mlx4_ib,ib_mad,nvmet_rdma,ib_umad ib_addr 16384 2 rdma_cm,ib_core mlx4_core 282624 2 mlx4_en,mlx4_ib vxlan 40960 1 mlx4_en configfs 36864 4 nvmet,target_core_mod,netconsole ptp 20480 2 e1000e,mlx4_en
mellanox infiniband mode
======================
ib_ipoib 77824 0 mlx4_ib 147456 0 ib_umad 24576 8 nvmet_rdma 24576 0 nvmet 24576 1 nvmet_rdma rdma_cm 45056 1 nvmet_rdma ib_cm 45056 2 rdma_cm,ib_ipoib ib_sa 36864 4 rdma_cm,ib_cm,mlx4_ib,ib_ipoib ib_mad 49152 4 ib_cm,ib_sa,mlx4_ib,ib_umad ib_core 114688 9 rdma_cm,ib_cm,ib_sa,iw_cm,mlx4_ib,ib_mad,nvmet_rdma,ib_umad,ib_ipoib ib_addr 16384 2 rdma_cm,ib_core nvme 61440 0 configfs 36864 4 nvmet,target_core_mod,netconsole mlx4_core 282624 1 mlx4_ib
#include <execinfo.h>
static void
dump_stack(void)
{
int j, nptrs;
#define SIZE 100
void *buffer[100];
char **strings;
nptrs = backtrace(buffer, SIZE);
strings = backtrace_symbols(buffer, nptrs);
if (strings == NULL) {
perror("backtrace_symbols");
exit(EXIT_FAILURE);
}
for (j = 0; j < nptrs; j++)
printf("%s\n", strings[j]);
printf("\n");
free(strings);
}
regular MMIO
============
nvme_mmio_write (qemu/hw/block/nvme.c:921)
memory_region_write_accessor (qemu/memory.c:451)
access_with_adjusted_size (qemu/memory.c:506)
memory_region_dispatch_write (qemu/memory.c:1158)
address_space_rw (qemu/exec.c:2547)
kvm_cpu_exec (qemu/kvm-all.c:1849)
qemu_kvm_cpu_thread_fn (qemu/cpus.c:1050)
start_thread (pthread_create.c:312)
clone
eventfd MMIO
==============
nvme_sq_notifier (qemu/hw/block/nvme.c:598)
aio_dispatch (qemu/aio-posix.c:329)
aio_ctx_dispatch (qemu/async.c:232)
g_main_context_dispatch
glib_pollfds_poll (qemu/main-loop.c:213)
os_host_main_loop_wait (qemu/main-loop.c:257)
main_loop_wait (qemu/main-loop.c:504)
main_loop (qemu/vl.c:1920)
main (qemu/vl.c:4682)
__libc_start_main
aio completion
==============
nvme_rw_cb (qemu-google/hw/block/nvme.c:244)
dma_complete (qemu-google/dma-helpers.c:115)
dma_blk_cb (qemu-google/dma-helpers.c:136)
bdrv_co_complete (qemu-google/block/io.c:2113)
bdrv_co_do_rw (qemu-google/block/io.c:2152)
coroutine_trampoline (qemu-google/util/coroutine-ucontext.c:81)
/lib/x86_64-linux-gnu/libc.so.6(+0x498b0) ../sysdeps/unix/sysv/linux/x86_64/__start_context.S:32
iothread
=========
handle_notify (qemu/hw/block/dataplane/virtio-blk.c:126)
aio_dispatch (qemu/aio-posix.c:329)
aio_poll (qemu/aio-posix.c:474)
iothread_run (qemu/iothread.c:45)
start_thread (pthread_create.c:312)
/lib/x86_64-linux-gnu/libc.so.6(clone+0x6d)
eth3: public IP
eth4: private IP
echo 1 > /proc/sys/net/ipv4/ip_forward
iptables -t nat -A POSTROUTING -o eth3 -j MASQUERADE
iptables -A FORWARD -i eth3 -o eth4 -m state –state RELATED,ESTABLISHED -j ACCEPT
iptables -A FORWARD -i eth4 -o eth3 -j ACCEPT
https://www.facebook.com/careers/jobs/a0IA000000CypMgMAJ/