How can I fix my characters movement speed

Question

I'm trying to make my first game, but I have a problem with the speed control:

Whenever I hold down on the button, everything slows down. I'm using time.deltatime which I know is global but I can't find any fixes.

void Start ()
{
    self = GetComponent<Rigidbody2D>();
    InvokeRepeating("SwitchDirections", switchTime, switchTime * 2);
    StartCoroutine(SpawnBallLoop());
    StartCoroutine(Count());
}

IEnumerator SpawnBallLoop ()
{
    while (gameOver == false)
    {
        yield return new WaitForSecondsRealtime(2f);
        SpawnBall();
    }
}

void SpawnBall ()
{
    Instantiate(ball, new Vector3(Random.Range(-2.18f, 2.18f), 4.6f, 0f), Quaternion.identity);
}

void UpdateClock ()
{
    seconds += 1;
    clock.text = "Time: " + seconds;
}

void SwitchDirections ()
{
    moveSpeed *= -1;
}

void FixedUpdate ()
{
    self.velocity = new Vector2(moveSpeed, 0f);
    if (Input.GetMouseButton(0))
    {
        Time.timeScale = 0.5f;
    }
    else
    {
        Time.timeScale = 1f;
    }
}

IEnumerator Count ()
{
    while (gameOver == false)
    {
        yield return new WaitForSecondsRealtime(1);
        UpdateClock();
    }
}

public void OnCollisionEnter2D(Collision2D collision)
{
    if (collision.transform.tag == "Ball")
    {
        GetComponent<SpriteRenderer>().enabled = false;
        transform.GetChild(0).gameObject.SetActive(true);
    }
}

Answer 1

  Time.timeScale = 0.5f;

TimeScale is the scale at which time passes. This can be used for slow motion effects. I suggest you to read this : here

Answer 2

Time.timeScale will slow down every game object in your scene. If you want only one game object to slow down use reduce movespeed of that one gameobject.