Isso finalmente funcionou fazendo algo assim:
location / {
proxy_pass_header Server;
proxy_set_header Host $http_host;
proxy_redirect off;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Scheme $scheme;
proxy_set_header X-Forwarded-Protocol ssl;
proxy_connect_timeout 120;
proxy_pass http://localhost:8000/;
location /admin/path/to/upload {
client_max_body_size 50m;
proxy_pass http://localhost:8000/admin/path/to/upload;
}
}