Remova o texto e mantenha a div dentro de um div.
$(function() {
$("#to-clean, #to-clean *") // selects the element and all element nodes inside it
.contents() // selects all child nodes including tags, comments and text
.filter(function() {
return this.nodeType === Node.TEXT_NODE; // filter text nodes
}).remove(); // boom!
});
Selim Soufargi