Clicking through a JLayeredPane

Question

I have a layeredPane with two panels (panel1 and panel2) on different layers. In panel2 is a text field. When panel1 is put above panel2 by clicking a button, panel1 disappears together with its textfield under panel2. However, when i click with the mouse on the place, where the text field was, it reappears. This should not happen! The only solution I found so far is to make the text field in panel2 invisible when panel1 is put above it. But this solution is not satisfying, because this can get complicated if there are more components within the different panels. How can I solve this problem elegantly?

I made an runnable example to show this behaviour:

import java.awt.Dimension;
import java.awt.FlowLayout;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import javax.swing.JButton;
import javax.swing.JLayeredPane;
import javax.swing.JPanel;
import javax.swing.JTextField;
import javax.swing.WindowConstants;

public class LayeredPaneTest2 extends javax.swing.JFrame {
    private final JButton button;
    private JLayeredPane layeredPane;
    private JPanel panel1;
    private JPanel panel2;
    private final JTextField textField;

    public LayeredPaneTest2() {
        setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE);
        getContentPane().setLayout(new FlowLayout());

        layeredPane = new JLayeredPane();
        panel1 = new JPanel();
        panel2 = new JPanel();
        textField = new JTextField();
        button = new JButton();

        // Config components.
        button.setText("Change Layers");
        button.addActionListener(new ActionListener() {
            @Override
            public void actionPerformed(ActionEvent evt) {
                layeredPane.setLayer(panel1, 1);
                layeredPane.setLayer(panel2, 0);
            }
        });
        textField.setText("test");
        layeredPane.setPreferredSize(new Dimension(300, 300));
        panel1.setBounds(0, 0, 300, 300);
        panel2.setBounds(0, 0, 300, 300);

        // Add components.
        panel2.add(textField);
        layeredPane.add(panel1);
        layeredPane.add(panel2);
        layeredPane.setLayer(panel1, 0);
        layeredPane.setLayer(panel2, 1);
        add(layeredPane);
        add(button);

        pack();
    }                                                           

    public static void main(String args[]) {
        java.awt.EventQueue.invokeLater(new Runnable() {
            public void run() {
                new LayeredPaneTest2().setVisible(true);
            }
        });
    }
}

Thank you for your help!