I came up with the following to fire a function when the RadUpload cancel button is pressed. Replace alert('cancel pressed') as required.
<telerik:radprogressarea id="progressArea1" runat="server" />
var cancelButton = $find('<%= progressArea1.ClientID %>')._findElement("CancelButton");
$addHandlers(cancelButton, { "click": function() { alert('cancel pressed') } }, this);