Why does my falling sand simulator have weird behaviour?

Question

I've been spending the last two days trying to figure out why my falling sand simulation does weird things it obviously shouldn't. I have moved from 2D Arrays to HashSets and finally to ConcurrentSkipListMap, with my own Point class as keys, and Element enums as values.

On the first look, and on a small scale, it would seem that it works just fine:
(I would have posted the gif's here, but they are too big)
https://i.stack.imgur.com/wUPyC.gif

But if you slow it down...
https://i.stack.imgur.com/Pd6NZ.gif

... you see that the sand falling to the left side of the pile gets teleported instantly to where it would end up. Though sand falling to the right side does smoothly "slide" down.

Also, my attempt at water doesn't look very good:
https://i.stack.imgur.com/2uq3q.gif

As I said, I already tried this with Arrays e.g. and even succeeded, the only problem being performance problems on larger scales. Like a 600x600 grid:
https://i.stack.imgur.com/X4wNk.gif

which completely glitches it, although it runs pretty smooth.
I first thought the floating sand was just random, but if you look closely, it only occurs when there is settled sand below it at the bottom.

Funny things happen when I now add water:
https://i.stack.imgur.com/E3zSe.gif

I just don't get what I did wrong and have tried everything I can think of. What did I miss?
Also, I know I haven't done everything in the most optimal way, and am not sure if ConcurrentSkipListMap is what I need. Until now I used ConcurrentHashMap but switched because it is not sorted. Should I stop using that?

This is the code in question: If you want me to provide the code in another way, please tell me!

Board.java

This class manages the map, steps through every particle and has a method for generating a bufferedImage, which GamePanel draws.

    public class Board {
    public ConcurrentSkipListMap<Point, Element> sMap;

    private final Game game;

    public Board(Game g) {
        this.game = g;
        sMap = new ConcurrentSkipListMap<>();

    }

    public void add(Point point, Element element) {
        if (outOfBounds(point)) {
            return;
        }
        sMap.put(point, element);
    }


    public void remove(Point point) {
        if (outOfBounds(point)) {
            return;
        }
        sMap.remove(point);
    }


    public boolean outOfBounds(Point point) {
        return outOfBounds(point.x, point.y);
    }

    public boolean outOfBounds(int x, int y) {
        return x < 0 || x >= game.gridSize || y < 0 || y >= game.gridSize;
    }

    public Element get(Point point) {
        if (outOfBounds(point)) {
            return STONE;
        }
        return sMap.getOrDefault(point, Element.VOID);
    }

    public void move(Point point, Element element, int dx, int dy) {
        if(dx != 0 || dy != 0) {
            sMap.remove(point,element);
            sMap.put(point.getTranslatedPoint(dx,dy), element);
        }
    }

    public void stepAll() {
        sMap.descendingMap().forEach((point, element) -> {
            int dx = 0;
            int dy = 0;
            if (!point.isUpdated) {
                switch (element) {
                    case SAND -> {

                        if (get(point.below()) == VOID || get(point.below()) == WATER) {
                            dy = 1;
                        } else if (get(point.below().right()) == VOID || get(point.below().right()) == WATER) {
                            dx = 1;
                            dy = 1;
                        } else if (get(point.below().left()) == VOID || get(point.below().left()) == WATER) {
                            dx = -1;
                            dy = 1;
                        }
                    }
                    case WATER -> {
                        if (get(point.below()) == VOID) {
                            dy = 1;
                        } else if (get(point.below().left()) == VOID) {
                            dx = 1;
                            dy = 1;
                        } else if (get(point.below().right()) == VOID) {
                            dx = -1;
                            dy = 1;
                        } else if (get(point.left()) == VOID) {
                            dx = 1;
                        } else if (get(point.right()) == VOID) {
                            dx = -1;
                        }
                    }
                }
                move(point, element, dx, dy);
            }
        });
    }


    public BufferedImage getNextFrame() {
        BufferedImage bImg = new BufferedImage(game.windowSize, game.windowSize, BufferedImage.TYPE_INT_ARGB);
        Graphics2D g2d = bImg.createGraphics();

        sMap.forEach((point, element) -> game.frame.panel.drawRect(g2d, element.color, point));

        g2d.dispose();
        return bImg;

    }

}

Point.java

    public class Point implements Comparable<Point> {
    public int x, y;

    public Point(int x, int y) {
        this.x = x;
        this.y = y;
    }

    @Override
    public int hashCode() {
        return Objects.hash(x, y);
    }

    @Override
    public boolean equals(Object o) {
        if (this == o) return true;
        if (o == null || getClass() != o.getClass()) return false;
        Point point2 = (Point) o;
        return x == point2.x && y == point2.y;
    }

    @Override
    public int compareTo(Point o) {
        return Integer.compare(this.hashCode(),o.hashCode());
    }

    public Point below() {
        return getTranslatedPoint(0, 1);
    }

    public Point left() {
        return getTranslatedPoint(-1, 0);
    }

    public Point right() {
        return getTranslatedPoint(1, 0);
    }

    public Point getTranslatedPoint(int dx, int dy) {
        return new Point(this.x + dx, this.y + dy);
    }

    public void translate(int dx, int dy) {
        this.x += dx;
        this.y += dy;
    }

    public void set(Point position) {
        this.x = position.x;
        this.y = position.y;
    }

    public void set(int x, int y) {
        this.x = x;
        this.y = y;
    }

    public String toString() {
        return "(x=" + x + "|y=" + y + ") ";
    }
}

Element.java

public enum Element {
    VOID,
    SAND (Color.yellow),
    WATER (Color.blue),
    STONE (Color.gray);

    public final Color color;

    Element(Color color) {
        this.color = color;
    }

    Element() {
        this.color = null;
    }

}

Game.java

This class actually had the stepping logic before but handles mostly input now.

public class Game {

    private final Game game;
    public final Board board;
    public final GameFrame frame;
    public final Loop loop;

    public final int cellSize;
    public final int windowSize;
    public final int gridSize;
    public int mouseX = 0;
    public int mouseY = 0;
    public int slot = 1;
    public int stepCnt = 0;
    boolean running = false;

    public Game(int windowSize, int gridSize) {
        this.windowSize = windowSize;
        this.gridSize = gridSize;
        this.cellSize = windowSize/gridSize;

        game = this;
        board = new Board(this);

        loop = new Loop(this);
        frame = new GameFrame(this);

        loop.run();
    }

    public static void main(String[] args) {
        int gridSize = 60;
        int windowSize = 600;
        new Game(windowSize,gridSize);
    }

    public void drawOnBoard(Action a, int x, int y) {
        Point mousePoint = new Point(x,y);
        switch (a) {
            case L_MOUSE:
                switch (slot) {
                    case 1 -> board.add(mousePoint, SAND);
                    case 2 -> board.add(mousePoint, WATER);
                    case 3 -> board.add(mousePoint, STONE);
                }

                break;
            case R_MOUSE:
                board.remove(mousePoint);

                break;
            case MID_MOUSE:
        }
    }


    public void step() {
        board.stepAll();
        stepCnt++;
    }


    public void onAction(Action a, AWTEvent e) {
        if (e instanceof MouseEvent) {
            mouseX = Math.floorDiv(((MouseEvent) e).getX(), game.cellSize);
            mouseY = Math.floorDiv(((MouseEvent) e).getY(), game.cellSize);
            // idek why i used math.floordiv

            drawOnBoard(a, mouseX, mouseY);


        } else if (e instanceof KeyEvent) {
            double increment = loop.stepDelay / 10;

            switch (a) {
                case SPACE -> game.toggleSimulation();
                case S -> game.step();
                case C -> {
                    board.sMap.clear();
                    stepCnt = 0;
                }
                case UP -> {
                    if (loop.stepDelay + increment <= 2000) {
                        loop.stepDelay += increment;
                    }
                }
                case DOWN -> {
                    if (loop.stepDelay - increment > 0) {
                        loop.stepDelay -= increment;

                    }
                }
                case RIGHT -> {
                    if (loop.stepsPerLoop <= 400000)
                        loop.stepsPerLoop += 1 + loop.stepsPerLoop / 10;
                }
                case LEFT -> {
                    if (loop.stepsPerLoop >= 0)
                        loop.stepsPerLoop -= 1 + loop.stepsPerLoop / 10;
                }
                case N1 -> this.slot = 1;
                case N2 -> this.slot = 2;
                case N3 -> this.slot = 3;
                case N4 -> this.slot = 4;
                case N5 -> board.sMap.descendingMap().forEach((key, value) -> System.out.println(key.toString() + ":" + value));
                case P -> System.out.println(board.sMap.size());
            }

        }
    }


    public void toggleSimulation() {
        running = !running;
        if (running) {
            startSimulation();
        } else {
            stopSimulation();
        }
    }

    public void startSimulation() {
        running = true;
        System.out.println("start");
    }

    public void stopSimulation() {
        running = false;
        System.out.println("stop");
    }
}

GameFrame.java

Just holds the GamePanel

public class GameFrame extends JFrame {
    Game game;
    public GamePanel panel;
    public InputHandler inputHandler;


    public GameFrame(Game g){
        this.game = g;
        panel = new GamePanel(game);

        this.setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE);
        this.setResizable(false);
        this.setVisible(true);


        this.getContentPane().add(panel);

        inputHandler = new InputHandler(game);
        this.getContentPane().addMouseListener(inputHandler);
        this.getContentPane().addMouseMotionListener(inputHandler);
        this.getContentPane().addMouseWheelListener(inputHandler);
        this.panel.addKeyListener(inputHandler);
        this.pack();

    }
}

GamePanel.java

Manages graphical stuff and renders the bufferedImage from Board.java

public class GamePanel extends JPanel {

    Game game;
    public GamePanel(Game g) {
        this.game = g;
        this.setBackground(Color.black);
        this.setFocusable(true);
        this.setPreferredSize(new Dimension(game.windowSize, game.windowSize));
    }
    @Override
    public void paintComponent(Graphics g) {
        super.paintComponent(g);
        draw((Graphics2D) g);
    }


    public void draw(Graphics2D g) {

        g.drawImage(game.board.getNextFrame(),0,0,this);
        if (game.cellSize > 5) {
            g.setColor(Color.darkGray);
            for (int i = 0; i <= game.gridSize; i++) {
                g.drawLine(game.cellSize * i, 0, game.cellSize * i, game.windowSize);
            }
            for (int i = 0; i <= game.gridSize; i++) {
                g.drawLine(0, game.cellSize * i, game.windowSize, game.cellSize * i);
            }
        }
    }

    public void drawRect(Graphics2D g, Color c, Point point) {
        drawRect(g, c, point, game.cellSize);
    }

    public void drawRect(Graphics2D g, Color c, Point point, int size) {
        Color prevCol = g.getColor();
        g.setColor(c);
        g.fillRect(point.x * game.cellSize, point.y * game.cellSize, size, size);
        g.setColor(prevCol);
    }

}

InputHandler.java

Basically just calls the onAction method in Game with Action enums.

public class InputHandler implements MouseListener, MouseMotionListener, MouseWheelListener, KeyListener {

    public int mouseX = 0;
    public int mouseY = 0;

    public boolean[] pressedButtons;
    public boolean[] pressedKeys;

    Game game;

    public InputHandler(Game g) {
        game = g;

        int buttons = java.awt.MouseInfo.getNumberOfButtons() + 1;
        pressedButtons = new boolean[buttons];
        pressedKeys = new boolean[3]; //shift = 0, strg = 1, alt = 2

    }

    @Override
    public void mousePressed(MouseEvent e) {

        pressedButtons[e.getButton()] = true;
        switch (e.getButton()) {
            case MouseEvent.BUTTON1 -> game.onAction(Action.L_MOUSE, e);
            case MouseEvent.BUTTON2 -> game.onAction(Action.MID_MOUSE, e);
            case MouseEvent.BUTTON3 -> game.onAction(Action.R_MOUSE, e);
        }
    }


    @Override
    public void mouseDragged(MouseEvent e) {
        mouseX = e.getX();
        mouseY = e.getY();

        if (pressedButtons[1]) {
            game.onAction(Action.L_MOUSE, e);
        }
        if (pressedButtons[2]) {
            game.onAction(Action.MID_MOUSE, e);
        }
        if (pressedButtons[3]) {
            game.onAction(Action.R_MOUSE, e);
        }
    }


    @Override
    public void mouseWheelMoved(MouseWheelEvent e) {
        if (e.getWheelRotation() < 0) {
            game.onAction(Action.WHEEL_UP, e);
        } else if (e.getWheelRotation() > 0) {
            game.onAction(Action.WHEEL_DOWN, e);
        }
    }

    @Override
    public void mouseReleased(MouseEvent e) {
        pressedButtons[e.getButton()] = false;
    }

    @Override
    public void keyPressed(KeyEvent e) {
        int keyCode = e.getKeyCode();

        switch (keyCode) {
            case KeyEvent.VK_SPACE -> game.onAction(Action.SPACE, e);
            case KeyEvent.VK_S -> game.onAction(Action.S, e);
            case KeyEvent.VK_C -> game.onAction(Action.C, e);
            case KeyEvent.VK_UP -> game.onAction(Action.UP, e);
            case KeyEvent.VK_DOWN -> game.onAction(Action.DOWN, e);
            case KeyEvent.VK_RIGHT -> game.onAction(Action.RIGHT, e);
            case KeyEvent.VK_LEFT -> game.onAction(Action.LEFT, e);
            case KeyEvent.VK_1 -> game.onAction(Action.N1, e);
            case KeyEvent.VK_2 -> game.onAction(Action.N2, e);
            case KeyEvent.VK_3 -> game.onAction(Action.N3, e);
            case KeyEvent.VK_4 -> game.onAction(Action.N4, e);
            case KeyEvent.VK_5 -> game.onAction(Action.N5, e);
            case KeyEvent.VK_P -> game.onAction(Action.P, e);
            case KeyEvent.VK_R -> game.onAction(Action.R, e);
        }


    }

    @Override
    public void keyReleased(KeyEvent e) {

    }

    @Override
    public void keyTyped(KeyEvent e) {

    }

    @Override
    public void mouseClicked(MouseEvent e) {

    }

    @Override
    public void mouseMoved(MouseEvent e) {

    }

    @Override
    public void mouseEntered(MouseEvent e) {

    }

    @Override
    public void mouseExited(MouseEvent e) {

    }

}

Loop.java

public class Loop {
    int stepsPerLoop = 1;
    boolean running = true;

    long startTime;
    int currentFrames;
    int fps;
    double stepDelay = 20;
    Game game;

    public Loop(Game g) {
        this.game = g;
    }
    public void run() {
        long lastloop = System.nanoTime();
        startTime = (lastloop / 1000000);
        long laststep = System.nanoTime();

        while (running) {
            long now = System.nanoTime();
            int saveHash = game.board.sMap.hashCode();
            if (game.running) {
                if ((now - laststep) / 1000000f >= stepDelay) {
                    for (int i = 0; i < stepsPerLoop; i++) {
                        game.step();

                    }
                    laststep = now;
                }
            }

            //frames which are the same es the frame before don't have to be rendered at 60 fps
            if ((now - lastloop) / 1000000f >= 15 && saveHash != game.board.sMap.hashCode()) {
                game.frame.repaint();
                game.frame.setTitle("Sand  (fps: " + fps + ")  STEP: " + game.stepCnt + "  STEPDELAY: " + stepDelay + "  STEPSPERLOOP: " + stepsPerLoop);
                countFrame();
                lastloop = now;
            } else if((now - lastloop) / 1000000f > 120){
                game.frame.repaint();
                game.frame.setTitle("Sand  (fps: " + fps + ")  STEP: " + game.stepCnt + "  STEPDELAY: " + stepDelay + "  STEPSPERLOOP: " + stepsPerLoop);
                countFrame();
                lastloop = now;
            }
        }
    }

    public void countFrame() {
        long now = System.currentTimeMillis();
        if (now - startTime >= 1000) {
            startTime = now;
            fps = currentFrames;
            currentFrames = 0;

        }
        currentFrames += 1;
    }


}

Action.java

public enum Action {
    L_MOUSE,
    R_MOUSE,
    MID_MOUSE,
    WHEEL_UP,
    WHEEL_DOWN,
    ALT,
    CTRL,
    SHIFT,
    SPACE,
    C,
    S,
    UP,
    DOWN,
    RIGHT,
    LEFT,
    N1,
    N2,
    N3,
    N4,
    N5,
    P,
    R;
}