Here are the code fragments I described above:
---------------- Create a key pair -------------------------------
RSACryptoServiceProvider csp = new RSACryptoServiceProvider(1024);

string s = csp.ToXmlString(true);
StreamWriter sw = new StreamWriter("KeyPriv.xml");
sw.WriteLine(s);
sw.Close();

s = csp.ToXmlString(false);
sw = new StreamWriter("KeyPub.xml");
sw.WriteLine(s);
sw.Close();

csp.Clear();
------------------------------------------------------------------
---------------- Encrypt with private key ------------------------
RSACryptoServiceProvider csp = new RSACryptoServiceProvider(1024);

StreamReader sr = new StreamReader("KeyPriv.xml");
string s = sr.ReadToEnd();
sr.Close();
csp.FromXmlString(s);

byte[] inp = System.Text.Encoding.Unicode.GetBytes(clearText);
byte[] outp = csp.Encrypt(inp, false);
cypherText = Convert.ToBase64String(outp);
csp.Clear();
------------------------------------------------------------------
---------------- Decrypt with public key -------------------------
RSACryptoServiceProvider csp = new RSACryptoServiceProvider(1024);

StreamReader sr = new StreamReader("KeyPub.xml");
string s = sr.ReadToEnd();
sr.Close();
csp.FromXmlString(s);

byte[] inp = Convert.FromBase64String(cypherText);
// This always throws an exception "invalid key" :(((
byte[] outp = csp.Decrypt(inp, false);
clearText = System.Text.Encoding.Unicode.GetString(outp);
csp.Clear();
------------------------------------------------------------------