I'm having two problems with my collision detection in XNA. There are two boxes, the red box represents a player and the blue box represents a wall.

The first problem is when the player moves to the upper side or bottom side of the wall and collides with it, and then try to go to the left or right, the player will just jump in the opposite direction as seen in the video.

However if I go to the right side or the left side of the wall and try to go up or down the player will smoothly go up or down without jumping.

The second problem is that when I collide with the box and my key is still pressed down the blue box goes half way through red box and and goes back out and it keeps doing that until I stop pressing the keyboard. its not very clear on the video but the keeps going in and out really fast until I stop pressing the key.

Here is a video example:- http://www.youtube.com/watch?v=mKLJsrPviYo

and Here is my code

Vector2 Position;
Rectangle PlayerRectangle, BoxRectangle;
float Speed = 0.25f;

enum Direction { Up, Right, Down, Left };
Direction direction;

protected override void Update(GameTime gameTime)
{
    // Allows the game to exit
    if (GamePad.GetState(PlayerIndex.One).Buttons.Back == ButtonState.Pressed)
        this.Exit();

    KeyboardState keyboardState = Keyboard.GetState();

   if (keyboardState.IsKeyDown(Keys.Up))
   {
       Position.Y -= (float)(Speed * gameTime.ElapsedGameTime.TotalMilliseconds);
       direction = Direction.Up;
   }

   if (keyboardState.IsKeyDown(Keys.Down))
   {
       Position.Y += (float)(Speed * gameTime.ElapsedGameTime.TotalMilliseconds);
       direction = Direction.Down;
   }

   if (keyboardState.IsKeyDown(Keys.Right))
   {
       Position.X += (float)(Speed * gameTime.ElapsedGameTime.TotalMilliseconds);
       direction = Direction.Right;
   }

   if (keyboardState.IsKeyDown(Keys.Left))
   {
        Position.X -= (float)(Speed * gameTime.ElapsedGameTime.TotalMilliseconds);
        direction = Direction.Left;
    }

    if (PlayerRectangle.Intersects(BoxRectangle))
    {
         if (direction == Direction.Right)
             Position.X = BoxRectangle.Left - PlayerRectangle.Width;
         else if (direction == Direction.Left)
             Position.X = BoxRectangle.Right;

         if (direction == Direction.Down)
             Position.Y = BoxRectangle.Top - PlayerRectangle.Height;
         else if (direction == Direction.Up)
             Position.Y = BoxRectangle.Bottom;
     }

     PlayerRectangle = new Rectangle((int)Position.X, (int)Position.Y, (int)32, (int)32);

     base.Update(gameTime);
}