Corrupção de memória do Apache no Strace

Question

Não tenho certeza se este é um problema SF ou SO.

Servidores LAMP muito (muito) ocupados no ambiente de carga balanceada Apache 2.2.26 no Centos 5.10.

Estou tentando rastrear um problema de código ou sistemas com processos httpd suspensos. Essas chamadas ficam em W, o status "Enviando Resposta" para sempre, as conexões tcp permanecem em atividade e o tempo limite do apache nunca é acionado. Eventualmente, acumulamos processos pendentes que eu tenho que devolver o processo do httpd.

Este é o fim de um strace que parece cercar todas as chamadas pendentes. Eu realmente não tenho certeza sobre onde ir em seguida com isso. Parece que o Apache está tentando escrever para o console, e não tenho certeza se isso é normal. Mas o erro malloc definitivamente aponta para algo errado. Qualquer ajuda (mesmo idéias malucas) é apreciada.

Não tenho certeza se existem moudles interessantes, mas aqui estão os LoadModules. LoadModule rewrite_module modules / mod_rewrite.so LoadModule info_module modules / mod_info.so LoadModule auth_digest_module modules / mod_auth_digest.so LoadModule proxy_module modules / mod_proxy.so LoadModule proxy_balancer_module modules / mod_proxy_balancer.so LoadModule proxy_connect_module modules / mod_proxy_connect.so LoadModule proxy_http_module modules / mod_proxy_http.so LoadModule ldap_module modules / mod_ldap.so LoadModule authnz_ldap_module modules / mod_authnz_ldap.so LoadModule deflate_module modules / mod_deflate.so LoadModule php4_module modules / libphp4.so

lstat("/tmp", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=12288, ...}) = 0

lstat("/tmp/promo.log", {st_mode=S_IFREG|0644, st_size=14778558, ...}) = 

open("/tmp/promo.log", O_WRONLY|O_CREAT|O_APPEND, 0666) = 27

fstat(27, {st_mode=S_IFREG|0644, st_size=14778558, ...}) = 0

lseek(27, 0, SEEK_CUR)                  = 0

lseek(27, 0, SEEK_CUR)                  = 0

write(27, "20140314065931 : cartitem::calcu"..., 68) = 68

close(27)                               = 0

open("/dev/tty", O_RDWR|O_NOCTTY|O_NONBLOCK) = -1 ENXIO (No such device or address)

writev(2, [{"*** glibc detected *** ", 23}, {"/usr/local/apache-2.2.26/bin/htt"..., 34}, {": ", 2}, {"double free or corruption (!prev"..., 33}, {": 0x", 4}, {"0000000005ebfb20", 16}, {" ***\n", 5}], 7) = 117

open("/usr/local/apache-2.2.26/lib/libgcc_s.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)

open("/usr/lib64/tls/libgcc_s.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)

open("/usr/lib64/libgcc_s.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)

open("/usr/local/apache-2.2.26/lib/libgcc_s.so.1", O_RDONLY) = -1 ENOENT (No such file or 
directory)

open("/etc/ld.so.cache", O_RDONLY)      = 27

fstat(27, {st_mode=S_IFREG|0644, st_size=93743, ...}) = 0

mmap(NULL, 93743, PROT_READ, MAP_PRIVATE, 27, 0) = 0x2ba82af8b000
close(27)                               = 0

open("/lib64/libgcc_s.so.1", O_RDONLY)  = 27

read(27, "7ELF
lstat("/tmp", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=12288, ...}) = 0

lstat("/tmp/promo.log", {st_mode=S_IFREG|0644, st_size=14778558, ...}) = 

open("/tmp/promo.log", O_WRONLY|O_CREAT|O_APPEND, 0666) = 27

fstat(27, {st_mode=S_IFREG|0644, st_size=14778558, ...}) = 0

lseek(27, 0, SEEK_CUR)                  = 0

lseek(27, 0, SEEK_CUR)                  = 0

write(27, "20140314065931 : cartitem::calcu"..., 68) = 68

close(27)                               = 0

open("/dev/tty", O_RDWR|O_NOCTTY|O_NONBLOCK) = -1 ENXIO (No such device or address)

writev(2, [{"*** glibc detected *** ", 23}, {"/usr/local/apache-2.2.26/bin/htt"..., 34}, {": ", 2}, {"double free or corruption (!prev"..., 33}, {": 0x", 4}, {"0000000005ebfb20", 16}, {" ***\n", 5}], 7) = 117

open("/usr/local/apache-2.2.26/lib/libgcc_s.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)

open("/usr/lib64/tls/libgcc_s.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)

open("/usr/lib64/libgcc_s.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)

open("/usr/local/apache-2.2.26/lib/libgcc_s.so.1", O_RDONLY) = -1 ENOENT (No such file or 
directory)

open("/etc/ld.so.cache", O_RDONLY)      = 27

fstat(27, {st_mode=S_IFREG|0644, st_size=93743, ...}) = 0

mmap(NULL, 93743, PROT_READ, MAP_PRIVATE, 27, 0) = 0x2ba82af8b000
close(27)                               = 0

open("/lib64/libgcc_s.so.1", O_RDONLY)  = 27

read(27, "7ELF%pre%%pre%%pre%%pre%%pre%%pre%%pre%%pre%%pre%%pre%>%pre%%pre%%pre%%pre%P'T8%pre%%pre%%pre%"..., 832) = 832

fstat(27, {st_mode=S_IFREG|0755, st_size=58400, ...}) = 0

open("/dev/tty", O_RDWR|O_NOCTTY|O_NONBLOCK) = -1 ENXIO (No such device or address)

writev(2, [{"*** glibc detected *** ", 23}, {"/usr/local/apache-2.2.26/bin/htt"..., 34}, {": ", 2}, {"malloc(): memory corruption", 27}, {": 0x", 4}, {"0000000005fc1e70", 16}, {" ***\n", 5}], 7) = 111

futex(0x2ba822f1dfc0, FUTEX_WAIT_PRIVATE, 9, NULL) = -1 EINTR (Interrupted system call)
--- SIGTERM (Terminated) @ 0 (0) ---

futex(0x2ba822f1b9e0, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EINTR (Interrupted system call)
--- SIGCONT (Continued) @ 0 (0) ---

futex(0x2ba822f1b9e0, FUTEX_WAIT_PRIVATE, 2, NULL
%pre%%pre%%pre%%pre%%pre%%pre%%pre%%pre%%pre%>%pre%%pre%%pre%%pre%P'T8%pre%%pre%%pre%"..., 832) = 832

fstat(27, {st_mode=S_IFREG|0755, st_size=58400, ...}) = 0

open("/dev/tty", O_RDWR|O_NOCTTY|O_NONBLOCK) = -1 ENXIO (No such device or address)

writev(2, [{"*** glibc detected *** ", 23}, {"/usr/local/apache-2.2.26/bin/htt"..., 34}, {": ", 2}, {"malloc(): memory corruption", 27}, {": 0x", 4}, {"0000000005fc1e70", 16}, {" ***\n", 5}], 7) = 111

futex(0x2ba822f1dfc0, FUTEX_WAIT_PRIVATE, 9, NULL) = -1 EINTR (Interrupted system call)
--- SIGTERM (Terminated) @ 0 (0) ---

futex(0x2ba822f1b9e0, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EINTR (Interrupted system call)
--- SIGCONT (Continued) @ 0 (0) ---

futex(0x2ba822f1b9e0, FUTEX_WAIT_PRIVATE, 2, NULL