I have been trying to upgrade our existing open source cassandra cluster from 3.0.9 version to 4.0.11. I have been following the standard OP for the upgrade, going node by node. But after upgrading the first node, I get the below error, even though the node joins the cluster and is normal. The client also sees constant failures while connecting to the node.
The cluster is a 2 Datacenter setup, both running 3.0.9 and 3 nodes on each DC. Do let me know if this rings a bell
ERROR [Messaging-EventLoop-3-1] 2023-08-01 06:15:49,140 InboundMessageHandler.java:300 - /x.x.x.x:7000->/x.x.x.x:7000-LEGACY_MESSAGES-e562c2ab unexpected exception caught while processing inbound messages; terminating connection
org.apache.cassandra.net.Crc$InvalidCrc: Read 1429076480, Computed -900387334
at org.apache.cassandra.net.InboundMessageHandler.processCorruptFrame(InboundMessageHandler.java:234)
at org.apache.cassandra.net.AbstractMessageHandler.process(AbstractMessageHandler.java:220)
at org.apache.cassandra.net.FrameDecoder.deliver(FrameDecoder.java:321)
at org.apache.cassandra.net.FrameDecoder.channelRead(FrameDecoder.java:285)
at org.apache.cassandra.net.FrameDecoder.channelRead(FrameDecoder.java:273)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
at org.apache.cassandra.net.FrameDecoderLegacyLZ4$LZ4Decoder.channelRead(FrameDecoderLegacyLZ4.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:795)
at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:480)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:378)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
INFO [Messaging-EventLoop-3-2] 2023-08-01 06:15:49,194 InboundConnectionInitiator.java:464 - /x.x.x.x:7000(/x..x.x.x:25217)->/x.x.x.x:7000-LEGACY_MESSAGES-0436552d messaging connection established, version = 10, framing = LZ4, encryption = unencrypted
INFO [Messaging-EventLoop-3-3] 2023-08-01 06:15:49,215 InboundConnectionInitiator.java:464 - /x.x.x.x:7000(/x.x.x.x:55335)->/x.x.x.x:7000-LEGACY_MESSAGES-664a3d54 messaging connection established, version = 10, framing = LZ4, encryption = unencrypted
ERROR [Messaging-EventLoop-3-3] 2023-08-01 06:15:49,215 InboundMessageHandler.java:182 - /x.x.x.x:7000->/x.x.x.x:7000-LEGACY_MESSAGES-664a3d54 unexpected exception caught while deserializing a message
org.apache.cassandra.net.InvalidSerializedSizeException: Invalid serialized size; expected 224, actual size at least 223, for verb READ_REQ
at org.apache.cassandra.net.InboundMessageHandler.processSmallMessage(InboundMessageHandler.java:170)
at org.apache.cassandra.net.InboundMessageHandler.processOneContainedMessage(InboundMessageHandler.java:151)
at org.apache.cassandra.net.AbstractMessageHandler.processFrameOfContainedMessages(AbstractMessageHandler.java:242)
at org.apache.cassandra.net.AbstractMessageHandler.processIntactFrame(AbstractMessageHandler.java:227)
at org.apache.cassandra.net.AbstractMessageHandler.process(AbstractMessageHandler.java:218)
at org.apache.cassandra.net.FrameDecoder.deliver(FrameDecoder.java:321)
at org.apache.cassandra.net.FrameDecoder.channelRead(FrameDecoder.java:285)
at org.apache.cassandra.net.FrameDecoder.channelRead(FrameDecoder.java:273)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
at org.apache.cassandra.net.FrameDecoderLegacyLZ4$LZ4Decoder.channelRead(FrameDecoderLegacyLZ4.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:795)
at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:480)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:378)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
ERROR [Messaging-EventLoop-3-3] 2023-08-01 06:15:49,215 InboundMessageHandler.java:300 - /x.x.x.x:7000->/x.x.x.x:7000-LEGACY_MESSAGES-664a3d54 unexpected exception caught while processing inbound messages; terminating connection
org.apache.cassandra.net.Crc$InvalidCrc: Read 1429076544, Computed -900387334
at org.apache.cassandra.net.InboundMessageHandler.processCorruptFrame(InboundMessageHandler.java:234)
at org.apache.cassandra.net.AbstractMessageHandler.process(AbstractMessageHandler.java:220)
at org.apache.cassandra.net.FrameDecoder.deliver(FrameDecoder.java:321)
at org.apache.cassandra.net.FrameDecoder.channelRead(FrameDecoder.java:285)
at org.apache.cassandra.net.FrameDecoder.channelRead(FrameDecoder.java:273)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
at org.apache.cassandra.net.FrameDecoderLegacyLZ4$LZ4Decoder.channelRead(FrameDecoderLegacyLZ4.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:795)
at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:480)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:378)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
trying: upgrade from the 3.0.9 to 4.0.11