Monthly Archives: November 2015

rdma modules

mellanox ethernet mode

nvmet_rdma             24576  0 
nvmet                  24576  1 nvmet_rdma
rdma_cm                45056  1 nvmet_rdma
ib_cm                  45056  1 rdma_cm
ib_umad                24576  0 
mlx4_ib               147456  0 
mlx4_en               110592  0 
ib_sa                  36864  3 rdma_cm,ib_cm,mlx4_ib
ib_mad                 49152  4 ib_cm,ib_sa,mlx4_ib,ib_umad
ib_core               114688  8 rdma_cm,ib_cm,ib_sa,iw_cm,mlx4_ib,ib_mad,nvmet_rdma,ib_umad
ib_addr                16384  2 rdma_cm,ib_core
mlx4_core             282624  2 mlx4_en,mlx4_ib
vxlan                  40960  1 mlx4_en
configfs               36864  4 nvmet,target_core_mod,netconsole
ptp                    20480  2 e1000e,mlx4_en

mellanox infiniband mode

ib_ipoib               77824  0 
mlx4_ib               147456  0 
ib_umad                24576  8 
nvmet_rdma             24576  0 
nvmet                  24576  1 nvmet_rdma
rdma_cm                45056  1 nvmet_rdma
ib_cm                  45056  2 rdma_cm,ib_ipoib
ib_sa                  36864  4 rdma_cm,ib_cm,mlx4_ib,ib_ipoib
ib_mad                 49152  4 ib_cm,ib_sa,mlx4_ib,ib_umad
ib_core               114688  9 rdma_cm,ib_cm,ib_sa,iw_cm,mlx4_ib,ib_mad,nvmet_rdma,ib_umad,ib_ipoib
ib_addr                16384  2 rdma_cm,ib_core
nvme                   61440  0 
configfs               36864  4 nvmet,target_core_mod,netconsole
mlx4_core             282624  1 mlx4_ib

rdma modules

qemu call stack

#include <execinfo.h>

static void
    int j, nptrs;
#define SIZE 100
    void *buffer[100];
    char **strings;

    nptrs = backtrace(buffer, SIZE);

    strings = backtrace_symbols(buffer, nptrs);
    if (strings == NULL) {

    for (j = 0; j < nptrs; j++)
        printf("%s\n", strings[j]);



regular MMIO
nvme_mmio_write (qemu/hw/block/nvme.c:921)
memory_region_write_accessor (qemu/memory.c:451)
access_with_adjusted_size (qemu/memory.c:506)
memory_region_dispatch_write (qemu/memory.c:1158)
address_space_rw (qemu/exec.c:2547)
kvm_cpu_exec (qemu/kvm-all.c:1849)
qemu_kvm_cpu_thread_fn (qemu/cpus.c:1050)
start_thread (pthread_create.c:312)

eventfd MMIO
nvme_sq_notifier (qemu/hw/block/nvme.c:598)
aio_dispatch (qemu/aio-posix.c:329)
aio_ctx_dispatch (qemu/async.c:232)
glib_pollfds_poll (qemu/main-loop.c:213)
os_host_main_loop_wait (qemu/main-loop.c:257)
main_loop_wait (qemu/main-loop.c:504)
main_loop (qemu/vl.c:1920)
main (qemu/vl.c:4682)

aio completion
nvme_rw_cb (qemu-google/hw/block/nvme.c:244)
dma_complete (qemu-google/dma-helpers.c:115)
dma_blk_cb (qemu-google/dma-helpers.c:136)
bdrv_co_complete (qemu-google/block/io.c:2113)
bdrv_co_do_rw (qemu-google/block/io.c:2152)
coroutine_trampoline (qemu-google/util/coroutine-ucontext.c:81)
/lib/x86_64-linux-gnu/ ../sysdeps/unix/sysv/linux/x86_64/__start_context.S:32

handle_notify (qemu/hw/block/dataplane/virtio-blk.c:126)
aio_dispatch (qemu/aio-posix.c:329)
aio_poll (qemu/aio-posix.c:474)
iothread_run (qemu/iothread.c:45)
start_thread (pthread_create.c:312)

configure linux as router

eth3: public IP

eth4: private IP

echo 1 > /proc/sys/net/ipv4/ip_forward
iptables -t nat -A POSTROUTING -o eth3 -j MASQUERADE
iptables -A FORWARD -i eth3 -o eth4 -m state –state RELATED,ESTABLISHED -j ACCEPT
iptables -A FORWARD -i eth4 -o eth3 -j ACCEPT

Software Engineer, Storage

Software Engineer, Storage

Facebook was built to help people connect and share, and over the last decade our tools have played a critical part in changing how people around the world communicate with one another. With over a billion people using the service and more than fifty offices around the globe, a career at Facebook offers countless ways to make an impact in a fast growing organization.
Facebook is seeking an experienced Software Engineer to join our storage team. The storage team is responsible for improving efficiencies by building more efficient systems for storing data as we grow from todays massive scale (hundreds of peta bytes) to a ridiculous scale (exa bytes). They are also responsible for providing distributed, fault-tolerant and cost effective storage technologies for various storage needs. The position is full-time (based in Menlo Park, CA) and there are minimal travel requirements.


  • Design and implement software improvements to software systems that provides data persistence (such as MySQL, Haystack, HDFS, HBase, and other key-value stores)
  • Analyze and improve reliability, availability, efficiency of Facebook’s distributed database and storage systems
  • Develop tools for testing, monitoring, and improving database/storage system performance, reliability and efficiency
  • Work closely with operation team to build operational friendly software system
  • Work with other teams to incorporate their innovations and vice versa


  • Bachelor’s Degree in Computer Science or related field
  • At least 2yrs developing and debugging in system level C/C++/Java, particularly on Linux
  • At least 2yrs with filesystems, concurrency, multithreading, server architectures, and distributed systems
  • Experience in developing distributed storage systems and servers
  • Experience working on database internals or multi-threaded server internals
  • Passionate about storage system and have good knowledge about trade offs between various storage media (disk, SSD devices, high end flash card) *Preferred