Este link tem a resposta e o fórum thread também é útil.
Basicamente, um exemplo com o Media Player Classic Home Cinema:
<playercorefactory>
<players>
<player name="MPC-HC" type="ExternalPlayer" audio="false" video="true">
<filename>C:\Program Files\MPC-HC\mplayerc.exe</filename>
<args>/fullscreen /close</args>
<hidexbmc>false</hidexbmc>
<hideconsole>false</hideconsole>
<hidecursor>false</hidecursor>
</player>
</players>
<rules action="prepend">
<rule filetypes="mkv" filename=".*720p.*" player="MPC-HC"/>
</rules>
</playercorefactory>