CRC with bit-wise operations in Dart

Question

I am having trouble with performing a CRC-64 ECMA calculation in Dart. While I am getting a result, it is not correct, as evidenced by a Python script that performs the same CRC calculation.

Here's the Python code:

def crc64ecma(data):
    """Calculate the CRC64-ECMA value for given data."""
    crc = 0xFFFFFFFFFFFFFFFF
    poly = 0x42EA3693F0E1EBA9

    print(f"[crc64ecma] Received data is {data.hex()}")  # Log the received data in hex format
    for byte in data:
        print(f"[crc64ecma] Byte is {byte}")
        crc ^= byte
        print(f"[crc64ecma] CRC is {crc:#018x}")
        for _ in range(8):
            if crc & 1:
                crc = (crc >> 1) ^ poly
                print(f"[crc64ecma] CRC is XOR with POLY as {crc:#018x}")
            else:
                crc >>= 1
                print(f"[crc64ecma] CRC is shifted as {crc:#018x}")
    crc = ~crc & 0xFFFFFFFFFFFFFFFF
    print(f"[crc64ecma] Final CRC is: {crc:#018x}")

    return crc


if __name__ == "__main__":
    while True:
        # Getting user input for 4 bytes
        user_input = input("Enter 4 bytes (in hexadecimal format, e.g., 1A2B3C4D): ")

        try:
            # Convert the input to bytes
            data = bytes.fromhex(user_input)

            # Ensure the input is exactly 4 bytes
            if len(data) != 4:
                raise ValueError("Input must be exactly 4 bytes")

            # Calculating CRC value
            crc_value = crc64ecma(data)

            # Printing the CRC result
            print(f"CRC: {crc_value:#018x}\n")  # Print CRC in hex format

        except ValueError as e:
            print(f"Error: {e}")

        # Option to continue or break the loop
        if input("Continue? (y/n): ").lower() != 'y':
            break

And for a input such as 0x65a58220 it yields the expected result of: 0xc179f267d045a14e

Here's my Dart version of such script:

import 'dart:developer';
import 'dart:typed_data';

const int POLY = 0x42EA3693F0E1EBA9;

Uint8List crc64ecma(Uint8List d) {
  var d2 = [101, 165, 130, 32];
  var data = Uint8List.fromList(d2);
  int crc = 0xFFFFFFFFFFFFFFFF;
  log('[crc64ecma] Received data is ${bytesToHex(data)}');
  for (var byte in data) {
    log('[crc64ecma] Byte is ${byte.toRadixString(16)}');
    crc ^= byte;
    log('[crc64ecma] CRC is ${crc.toRadixString(16)}');
    for (int i = 0; i < 8; i++) {
      if (crc & 1 != 0) {
        crc = (crc >> 1) ^ POLY;
        log('[crc64ecma] CRC is XOR with POLY as ${crc.toRadixString(16)}');
      } else {
        crc >>= 1;
        log('[crc64ecma] CRC is shifted as ${crc.toRadixString(16)}');
      }
    }
  }
  crc = ~crc & 0xFFFFFFFFFFFFFFFF;

  ByteData byteData = ByteData(8); 
  byteData.setUint64(0, crc, Endian.big);
  log('[crc64ecma] Final CRC is: ${bytesToHex(byteData.buffer.asUint8List())}');
  return byteData.buffer.asUint8List();
}

String bytesToHex(Uint8List bytes) {
  return bytes.map((byte) => byte.toRadixString(16)).join();
}

The Dart script yields a CRC of 0x3e86d98d045a14e.

So what appears to be the problem? The handling of bit-wise operations.

I have the logs for each step of the algorithm. Python logs this:

[crc64ecma] CRC is 0xffffffffffffff9a

While Dart logs this:

[crc64ecma] CRC is -66 // 0xFF9A

It appears that Dart's bitwise operation handles bytes with different sizes differently, truncating the CRC value. What could I do to force Dart to use the full width of the operator?

Answer 1

You can split your 64-bit integers into two, representing the high and low 32 bits:

import 'dart:typed_data';

const int POLY_HIGH = 0x42EA3693;
const int POLY_LOW = 0xF0E1EBA9;

Uint8List crc64ecma(Uint8List data) {
  int crcHigh = 0xFFFFFFFF;
  int crcLow = 0xFFFFFFFF;
  for (var byte in data) {
    crcLow ^= byte;
    for (int i = 0; i < 8; i++) {
      int lowBit = crcLow & 1;
      
      crcLow = (crcHigh & 1) << 31 | crcLow >>> 1;
      crcHigh >>>= 1;
      
      if (lowBit != 0) {
        crcLow ^= POLY_LOW;
        crcHigh ^= POLY_HIGH;
      }
    }
  }
  crcHigh = ~crcHigh;
  crcLow = ~crcLow;

  ByteData byteData = ByteData(8); 
  byteData.setUint32(0, crcHigh, Endian.big);
  byteData.setUint32(4, crcLow, Endian.big);
  return byteData.buffer.asUint8List();
}

String bytesToHex(Uint8List bytes) {
  return bytes.map((byte) => byte.toRadixString(16)).join();
}

void main() {
  print(bytesToHex(crc64ecma(Uint8List.fromList([101, 165, 130, 32]))));
}

Answer 2

I had to resort to using Dart's Big Int implementation, that way the values are not truncated:

Uint8List crc64ecma(Uint8List data) {
  BigInt crc = BigInt.parse('FFFFFFFFFFFFFFFF', radix: 16);
  log('[crc64ecma] Received data is ${bytesToHex(data)}');

  for (var byte in data) {
    log('[crc64ecma] Byte is ${byte.toRadixString(16)}');
    crc ^= BigInt.from(byte);
    log('[crc64ecma] CRC is ${crc.toRadixString(16)}');

    for (int i = 0; i < 8; i++) {
      if ((crc & BigInt.from(1)) != BigInt.zero) {
        crc = (crc >> 1) ^ BigInt.from(POLY);
      } else {
        crc >>= 1;
      }
      log('[crc64ecma] CRC after operation is ${crc.toRadixString(16)}');
    }
  }
  crc = ~crc & BigInt.parse('FFFFFFFFFFFFFFFF', radix: 16);

  
  var res = bigIntToUint8List(crc);

  log('[crc64ecma] Final CRC is: ${  crc.toRadixString(16)}');
  return res;
}

Uint8List bigIntToUint8List(BigInt bigInt) {
  var byteList = bigInt.toRadixString(16).padLeft(16, '0');

  // Convert hex string to byte array
  List<int> byteArray = [];
  for (int i = 0; i < byteList.length; i += 2) {
    String hexByte = byteList.substring(i, i + 2);
    byteArray.add(int.parse(hexByte, radix: 16));
  }

  // Convert byte array to Uint8List
  return Uint8List.fromList(byteArray);
}