Eu consegui trabalhar com o filetype.vim
agora:
" ~/.vim/filetype.vim
if exists("did_load_filetypes")
finish
endif
function! DetectOctave()
echom getline(1)
if getline(1) =~ '^#!/.*octave.*'
set filetype=octave
endif
endfunction
augroup filetypedetect
au! BufRead,BufNewFile *.m call DetectOctave()
augroup END