Eu tenho uma máquina ( test-server
) com um servidor rabbitmq e 4 trabalhadores aipo, e outra máquina ( test-worker
) com 240 trabalhadores aipo, que se conectam ao servidor rabbitmq em test-server
.
Todas as filas estão vazias no momento.
Com essa configuração, beam.smp
(que eu entendo é um processo relacionado a rabbitmq) vai de 200-250% da CPU e consome algumas centenas de MBs de RAM (isso pode ser OK, não tenho certeza).
Se eu parar os trabalhadores na máquina remota, ela volta ao normal. Se eu apenas começar por exemplo 40 trabalhadores em vez de 240, então é mais ou menos OK - ainda consome CPU, mas em torno de 50%.
O thread principal beam.smp está preso em select
, o que eu acho que está OK, já que está apenas ouvindo os threads filhos. Abaixo está um strace dos threads filho. Há algumas chamadas para epoll_wait
com tempo limite zero e também muitas futex
chamadas.
Eu também encontrei este bug, descrito em oslo (não sei o que é isso) link , que também menciona zero-timeout epoll_wait
chamadas e menciona rabbitmq.
Alguma ideia se este é o comportamento esperado do coelho nestas condições? Onde devo procurar a causa?
Obrigado
test-server$ sudo strace -p 26866 2>&1 | head -n 50
Process 26866 attached
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
epoll_wait(3, {}, 256, 0) = 0
clock_gettime(CLOCK_MONOTONIC, {87999, 785829269}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
writev(473, [{NULL, 0}, {"Process 26867 attached
clock_gettime(CLOCK_MONOTONIC, {88350, 863599878}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x82e500, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
clock_gettime(CLOCK_MONOTONIC, {88350, 865231792}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 865436250}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 865776903}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 872757864}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 872984686}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 873209787}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 873382297}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 873578979}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
epoll_wait(3, {}, 256, 0) = 0
clock_gettime(CLOCK_MONOTONIC, {88350, 875428570}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 875624976}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 875847357}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 876478262}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x82e500, FUTEX_WAIT_PRIVATE, 2, NULL) = 0
test-server$ sudo strace -p 26866 2>&1 | head -n 50
Process 26866 attached
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
epoll_wait(3, {}, 256, 0) = 0
clock_gettime(CLOCK_MONOTONIC, {87999, 785829269}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
writev(473, [{NULL, 0}, {"Process 26867 attached
clock_gettime(CLOCK_MONOTONIC, {88350, 863599878}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x82e500, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
clock_gettime(CLOCK_MONOTONIC, {88350, 865231792}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 865436250}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 865776903}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 872757864}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 872984686}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 873209787}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 873382297}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 873578979}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
epoll_wait(3, {}, 256, 0) = 0
clock_gettime(CLOCK_MONOTONIC, {88350, 875428570}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 875624976}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 875847357}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {88350, 876478262}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x82e500, FUTEX_WAIT_PRIVATE, 2, NULL) = 0
%pre%%pre%%pre%-%pre%<%pre%<None3%pre%%pre%%pre%%pre%%pre%%pre%6%pre%celer"..., 72}, {"0%pre%application/jsonutf-8%pre%%pre%%pre%*ho"..., 73}, {"6%pre%%pre%%pre%#", 8}, {"{\"sw_sys\": \"Linux\", \"clock\": 136"..., 291}, {"6", 1}], 6) = 445
clock_gettime(CLOCK_MONOTONIC, {87999, 786592082}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
epoll_wait(3, {}, 256, 0) = 0
clock_gettime(CLOCK_MONOTONIC, {87999, 787427449}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
epoll_wait(3, {}, 256, 0) = 0
clock_gettime(CLOCK_MONOTONIC, {87999, 788308663}) = 0
writev(201, [{NULL, 0}, {"%pre%%pre%%pre%%pre%-%pre%<%pre%<None2%pre%%pre%%pre%%pre%%pre%%pre%5%pre%celer"..., 72}, {"0%pre%application/jsonutf-8%pre%%pre%%pre%*ho"..., 73}, {"6%pre%%pre%%pre%#", 8}, {"{\"sw_sys\": \"Linux\", \"clock\": 136"..., 291}, {"6", 1}], 6) = 445
clock_gettime(CLOCK_MONOTONIC, {87999, 789017598}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 0
clock_gettime(CLOCK_MONOTONIC, {87999, 789278489}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
writev(392, [{NULL, 0}, {"%pre%%pre%%pre%%pre%-%pre%<%pre%<None3%pre%%pre%%pre%%pre%%pre%%pre%0%pre%celer"..., 72}, {"0%pre%application/jsonutf-8%pre%%pre%%pre%*ho"..., 73}, {"6%pre%%pre%%pre%#", 8}, {"{\"sw_sys\": \"Linux\", \"clock\": 136"..., 291}, {"6", 1}], 6) = 445
clock_gettime(CLOCK_MONOTONIC, {87999, 792374556}) = 0
clock_gettime(CLOCK_MONOTONIC, {87999, 792553480}) = 0
clock_gettime(CLOCK_MONOTONIC, {87999, 792796024}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {87999, 793154206}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {87999, 793493003}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {87999, 793842449}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {87999, 794054061}) = 0
writev(318, [{NULL, 0}, {"%pre%%pre%%pre%%pre%-%pre%<%pre%<None2%pre%%pre%%pre%%pre%%pre%%pre%0%pre%celer"..., 72}, {"0%pre%application/jsonutf-8%pre%%pre%%pre%*ho"..., 73}, {"6%pre%%pre%%pre%#", 8}, {"{\"sw_sys\": \"Linux\", \"clock\": 136"..., 291}, {"6%pre%%pre%%pre%%pre%-%pre%<%pre%<None2%pre%%pre%%pre%%pre%%pre%%pre%1%pre%cele"..., 73}, {"0%pre%application/jsonutf-8%pre%%pre%%pre%*ho"..., 73}, {"6%pre%%pre%%pre%#", 8}, {"{\"sw_sys\": \"Linux\", \"clock\": 136"..., 291}, {"6", 1}], 10) = 890
clock_gettime(CLOCK_MONOTONIC, {87999, 794411001}) = 0
clock_gettime(CLOCK_MONOTONIC, {87999, 795090977}) = 0
epoll_wait(3, {}, 256, 0) = 0
clock_gettime(CLOCK_MONOTONIC, {87999, 796129182}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
%pre%%pre%-%pre%<%pre%<None3%pre%%pre%%pre%%pre%%pre%%pre%6%pre%celer"..., 72}, {"0%pre%application/jsonutf-8%pre%%pre%%pre%*ho"..., 73}, {"6%pre%%pre%%pre%#", 8}, {"{\"sw_sys\": \"Linux\", \"clock\": 136"..., 291}, {"6", 1}], 6) = 445
clock_gettime(CLOCK_MONOTONIC, {87999, 786592082}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
epoll_wait(3, {}, 256, 0) = 0
clock_gettime(CLOCK_MONOTONIC, {87999, 787427449}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
epoll_wait(3, {}, 256, 0) = 0
clock_gettime(CLOCK_MONOTONIC, {87999, 788308663}) = 0
writev(201, [{NULL, 0}, {"%pre%%pre%%pre%%pre%-%pre%<%pre%<None2%pre%%pre%%pre%%pre%%pre%%pre%5%pre%celer"..., 72}, {"0%pre%application/jsonutf-8%pre%%pre%%pre%*ho"..., 73}, {"6%pre%%pre%%pre%#", 8}, {"{\"sw_sys\": \"Linux\", \"clock\": 136"..., 291}, {"6", 1}], 6) = 445
clock_gettime(CLOCK_MONOTONIC, {87999, 789017598}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 0
clock_gettime(CLOCK_MONOTONIC, {87999, 789278489}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
writev(392, [{NULL, 0}, {"%pre%%pre%%pre%%pre%-%pre%<%pre%<None3%pre%%pre%%pre%%pre%%pre%%pre%0%pre%celer"..., 72}, {"0%pre%application/jsonutf-8%pre%%pre%%pre%*ho"..., 73}, {"6%pre%%pre%%pre%#", 8}, {"{\"sw_sys\": \"Linux\", \"clock\": 136"..., 291}, {"6", 1}], 6) = 445
clock_gettime(CLOCK_MONOTONIC, {87999, 792374556}) = 0
clock_gettime(CLOCK_MONOTONIC, {87999, 792553480}) = 0
clock_gettime(CLOCK_MONOTONIC, {87999, 792796024}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {87999, 793154206}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {87999, 793493003}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {87999, 793842449}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
clock_gettime(CLOCK_MONOTONIC, {87999, 794054061}) = 0
writev(318, [{NULL, 0}, {"%pre%%pre%%pre%%pre%-%pre%<%pre%<None2%pre%%pre%%pre%%pre%%pre%%pre%0%pre%celer"..., 72}, {"0%pre%application/jsonutf-8%pre%%pre%%pre%*ho"..., 73}, {"6%pre%%pre%%pre%#", 8}, {"{\"sw_sys\": \"Linux\", \"clock\": 136"..., 291}, {"6%pre%%pre%%pre%%pre%-%pre%<%pre%<None2%pre%%pre%%pre%%pre%%pre%%pre%1%pre%cele"..., 73}, {"0%pre%application/jsonutf-8%pre%%pre%%pre%*ho"..., 73}, {"6%pre%%pre%%pre%#", 8}, {"{\"sw_sys\": \"Linux\", \"clock\": 136"..., 291}, {"6", 1}], 10) = 890
clock_gettime(CLOCK_MONOTONIC, {87999, 794411001}) = 0
clock_gettime(CLOCK_MONOTONIC, {87999, 795090977}) = 0
epoll_wait(3, {}, 256, 0) = 0
clock_gettime(CLOCK_MONOTONIC, {87999, 796129182}) = 0
futex(0x82e500, FUTEX_WAKE_PRIVATE, 1) = 1
Outro trecho:
%pre%