More precisely, suppose: - * - *
This class is NOT thread safe.
- */
-public class OrderedCode {
- // We want to encode a few extra symbols in strings:
- // For better performance, it uses the input array provided (not a copy). Therefore the
- * input array should not be modified.
- */
- public OrderedCode(byte[] encodedByteArray) {
- encodedArrays.add(encodedByteArray);
- }
-
- /**
- * Adds the given byte array item to the OrderedCode. It encodes the input byte array, followed by
- * a separator and appends the result to its internal encoded byte array store.
- *
- * It works with the input array, so the input array 'value' should not be modified till the
- * method returns.
- *
- * @param value bytes to be written.
- * @see #readBytes()
- */
- public void writeBytes(byte[] value) {
- // Determine the length of the encoded array
- int encodedLength = 2; // for separator
- for (byte b : value) {
- if ((b == ESCAPE1) || (b == ESCAPE2)) {
- encodedLength += 2;
- } else {
- encodedLength++;
- }
- }
-
- byte[] encodedArray = new byte[encodedLength];
- int copyStart = 0;
- int outIndex = 0;
- for (int i = 0; i < value.length; i++) {
- byte b = value[i];
- if (b == ESCAPE1) {
- System.arraycopy(value, copyStart, encodedArray, outIndex, i - copyStart);
- outIndex += i - copyStart;
- encodedArray[outIndex++] = ESCAPE1;
- encodedArray[outIndex++] = NULL_CHARACTER;
- copyStart = i + 1;
- } else if (b == ESCAPE2) {
- System.arraycopy(value, copyStart, encodedArray, outIndex, i - copyStart);
- outIndex += i - copyStart;
- encodedArray[outIndex++] = ESCAPE2;
- encodedArray[outIndex++] = FF_CHARACTER;
- copyStart = i + 1;
- }
- }
- if (copyStart < value.length) {
- System.arraycopy(value, copyStart, encodedArray, outIndex, value.length - copyStart);
- outIndex += value.length - copyStart;
- }
- encodedArray[outIndex++] = ESCAPE1;
- encodedArray[outIndex] = SEPARATOR;
-
- encodedArrays.add(encodedArray);
- }
-
- /**
- * Encodes the long item, in big-endian format, and appends the result to its internal encoded
- * byte array store.
- *
- * Note that the specified long is treated like a uint64, e.g. {@code new
- * OrderedCode().writeNumIncreasing(-1L).getEncodedBytes()} is greater than {@code new
- * OrderedCode().writeNumIncreasing(Long.MAX_VALUE).getEncodedBytes()}.
- *
- * @see #readNumIncreasing()
- */
- public void writeNumIncreasing(long value) {
- // Values are encoded with a single byte length prefix, followed
- // by the actual value in big-endian format with leading 0 bytes
- // dropped.
- byte[] bufer = new byte[9]; // 8 bytes for value plus one byte for length
- int len = 0;
- while (value != 0) {
- len++;
- bufer[9 - len] = (byte) (value & 0xff);
- value >>>= 8;
- }
- bufer[9 - len - 1] = (byte) len;
- len++;
- byte[] encodedArray = new byte[len];
- System.arraycopy(bufer, 9 - len, encodedArray, 0, len);
- encodedArrays.add(encodedArray);
- }
-
- /** Return floor(log2(n)) for positive integer n. Returns -1 iff n == 0. */
- int log2Floor(long n) {
- if (n < 0) {
- throw new IllegalArgumentException("must be non-negative");
- }
- return n == 0 ? -1 : LongMath.log2(n, RoundingMode.FLOOR);
- }
-
- /** Calculates the encoding length in bytes of the signed number n. */
- int getSignedEncodingLength(long n) {
- return BITS_TO_LENGTH[log2Floor(n < 0 ? ~n : n) + 1];
- }
-
- /**
- * Encodes the long item, in big-endian format, and appends the result to its internal encoded
- * byte array store.
- *
- * Note that the specified long is treated like an int64, i.e. {@code new
- * OrderedCode().writeNumIncreasing(-1L).getEncodedBytes()} is less than {@code new
- * OrderedCode().writeNumIncreasing(0L).getEncodedBytes()}.
- *
- * @see #readSignedNumIncreasing()
- */
- public void writeSignedNumIncreasing(long val) {
- long x = val < 0 ? ~val : val;
- if (x < 64) { // Fast path for encoding length == 1.
- byte[] encodedArray = new byte[] {(byte) (LENGTH_TO_HEADER_BITS[1][0] ^ val)};
- encodedArrays.add(encodedArray);
- return;
- }
- // buf = val in network byte order, sign extended to 10 bytes.
- byte signByte = val < 0 ? (byte) 0xff : 0;
- byte[] buf = new byte[2 + Longs.BYTES];
- buf[0] = buf[1] = signByte;
- System.arraycopy(Longs.toByteArray(val), 0, buf, 2, Longs.BYTES);
- int len = getSignedEncodingLength(x);
- if (len < 2) {
- throw new IllegalStateException(
- "Invalid length (" + len + ")" + " returned by getSignedEncodingLength(" + x + ")");
- }
- int beginIndex = buf.length - len;
- buf[beginIndex] ^= LENGTH_TO_HEADER_BITS[len][0];
- buf[beginIndex + 1] ^= LENGTH_TO_HEADER_BITS[len][1];
-
- byte[] encodedArray = new byte[len];
- System.arraycopy(buf, beginIndex, encodedArray, 0, len);
- encodedArrays.add(encodedArray);
- }
-
- /**
- * Encodes and appends INFINITY item to its internal encoded byte array store.
- *
- * @see #readInfinity()
- */
- public void writeInfinity() {
- writeTrailingBytes(INFINITY_ENCODED);
- }
-
- /**
- * Appends the byte array item to its internal encoded byte array store. This is used for the last
- * item and is not encoded. It also can be used to write a fixed number of bytes that will be read
- * back using {@link #readBytes(int)}.
- *
- * It stores the input array in the store, so the input array 'value' should not be modified.
- *
- * @param value bytes to be written.
- * @see #readTrailingBytes()
- * @see #readBytes(int)
- */
- public void writeTrailingBytes(byte[] value) {
- if ((value == null) || (value.length == 0)) {
- throw new IllegalArgumentException("Value cannot be null or have 0 elements");
- }
-
- encodedArrays.add(value);
- }
-
- /**
- * Returns the next byte array item from its encoded byte array store and removes the item from
- * the store.
- *
- * @see #writeBytes(byte[])
- */
- public byte[] readBytes() {
- if ((encodedArrays == null)
- || encodedArrays.isEmpty()
- || (encodedArrays.get(0).length - firstArrayPosition <= 0)) {
- throw new IllegalArgumentException("Invalid encoded byte array");
- }
-
- // Determine the length of the decoded array
- // We only scan up to "length-2" since a valid string must end with
- // a two character terminator: 'ESCAPE1 SEPARATOR'
- byte[] store = encodedArrays.get(0);
- int decodedLength = 0;
- boolean valid = false;
- int i = firstArrayPosition;
- while (i < store.length - 1) {
- byte b = store[i++];
- if (b == ESCAPE1) {
- b = store[i++];
- if (b == SEPARATOR) {
- valid = true;
- break;
- } else if (b == NULL_CHARACTER) {
- decodedLength++;
- } else {
- throw new IllegalArgumentException("Invalid encoded byte array");
- }
- } else if (b == ESCAPE2) {
- b = store[i++];
- if (b == FF_CHARACTER) {
- decodedLength++;
- } else {
- throw new IllegalArgumentException("Invalid encoded byte array");
- }
- } else {
- decodedLength++;
- }
- }
- if (!valid) {
- throw new IllegalArgumentException("Invalid encoded byte array");
- }
-
- byte[] decodedArray = new byte[decodedLength];
- int copyStart = firstArrayPosition;
- int outIndex = 0;
- int j = firstArrayPosition;
- while (j < store.length - 1) {
- byte b = store[j++]; // note that j has been incremented
- if (b == ESCAPE1) {
- System.arraycopy(store, copyStart, decodedArray, outIndex, j - copyStart - 1);
- outIndex += j - copyStart - 1;
- // ESCAPE1 SEPARATOR ends component
- // ESCAPE1 NULL_CHARACTER represents '\0'
- b = store[j++];
- if (b == SEPARATOR) {
- if ((store.length - j) == 0) {
- // we are done with the first array
- encodedArrays.remove(0);
- firstArrayPosition = 0;
- } else {
- firstArrayPosition = j;
- }
- return decodedArray;
- } else if (b == NULL_CHARACTER) {
- decodedArray[outIndex++] = 0x00;
- } // else not required - handled during length determination
- copyStart = j;
- } else if (b == ESCAPE2) {
- System.arraycopy(store, copyStart, decodedArray, outIndex, j - copyStart - 1);
- outIndex += j - copyStart - 1;
- // ESCAPE2 FF_CHARACTER represents '\xff'
- // ESCAPE2 INFINITY is an error
- b = store[j++];
- if (b == FF_CHARACTER) {
- decodedArray[outIndex++] = (byte) 0xff;
- } // else not required - handled during length determination
- copyStart = j;
- }
- }
- // not required due to the first phase, but need to entertain the compiler
- throw new IllegalArgumentException("Invalid encoded byte array");
- }
-
- /**
- * Returns the next long item (encoded in big-endian format via {@code writeNumIncreasing(long)})
- * from its internal encoded byte array store and removes the item from the store.
- *
- * @see #writeNumIncreasing(long)
- */
- public long readNumIncreasing() {
- if ((encodedArrays == null)
- || encodedArrays.isEmpty()
- || (encodedArrays.get(0).length - firstArrayPosition < 1)) {
- throw new IllegalArgumentException("Invalid encoded byte array");
- }
-
- byte[] store = encodedArrays.get(0);
- // Decode length byte
- int len = store[firstArrayPosition];
- if ((firstArrayPosition + len + 1 > store.length) || len > 8) {
- throw new IllegalArgumentException("Invalid encoded byte array");
- }
-
- long result = 0;
- for (int i = 0; i < len; i++) {
- result <<= 8;
- result |= (store[firstArrayPosition + i + 1] & 0xff);
- }
-
- if ((store.length - firstArrayPosition - len - 1) == 0) {
- // we are done with the first array
- encodedArrays.remove(0);
- firstArrayPosition = 0;
- } else {
- firstArrayPosition = firstArrayPosition + len + 1;
- }
-
- return result;
- }
-
- /**
- * Returns the next long item (encoded via {@code writeSignedNumIncreasing(long)}) from its
- * internal encoded byte array store and removes the item from the store.
- *
- * @see #writeSignedNumIncreasing(long)
- */
- public long readSignedNumIncreasing() {
- if ((encodedArrays == null)
- || encodedArrays.isEmpty()
- || (encodedArrays.get(0).length - firstArrayPosition < 1)) {
- throw new IllegalArgumentException("Invalid encoded byte array");
- }
-
- byte[] store = encodedArrays.get(0);
-
- long xorMask = ((store[firstArrayPosition] & 0x80) == 0) ? ~0L : 0L;
- // Store first byte as an int rather than a (signed) byte -- to avoid
- // accidental byte-to-int promotion later, which would extend the byte's
- // sign bit (if any).
- int firstByte = (store[firstArrayPosition] & 0xff) ^ (int) (xorMask & 0xff);
-
- // Now calculate and test length, and set x to raw (unmasked) result.
- int len;
- long x;
- if (firstByte != 0xff) {
- len = 7 - log2Floor(firstByte ^ 0xff);
- if (store.length - firstArrayPosition < len) {
- throw new IllegalArgumentException("Invalid encoded byte array");
- }
- x = xorMask; // Sign extend using xorMask.
- for (int i = firstArrayPosition; i < firstArrayPosition + len; i++) {
- x = (x << 8) | (store[i] & 0xff);
- }
- } else {
- len = 8;
- if (store.length - firstArrayPosition < len) {
- throw new IllegalArgumentException("Invalid encoded byte array");
- }
- int secondByte = (store[firstArrayPosition + 1] & 0xff) ^ (int) (xorMask & 0xff);
- if (secondByte >= 0x80) {
- if (secondByte < 0xc0) {
- len = 9;
- } else {
- int thirdByte = (store[firstArrayPosition + 2] & 0xff) ^ (int) (xorMask & 0xff);
- if (secondByte == 0xc0 && thirdByte < 0x80) {
- len = 10;
- } else {
- // Either len > 10 or len == 10 and #bits > 63.
- throw new IllegalArgumentException("Invalid encoded byte array");
- }
- }
- if (store.length - firstArrayPosition < len) {
- throw new IllegalArgumentException("Invalid encoded byte array");
- }
- }
- x =
- Longs.fromByteArray(
- Arrays.copyOfRange(store, firstArrayPosition + len - 8, firstArrayPosition + len));
- }
-
- x ^= LENGTH_TO_MASK[len]; // Remove spurious header bits.
-
- if (len != getSignedEncodingLength(x)) {
- throw new IllegalArgumentException("Invalid encoded byte array");
- }
-
- if ((store.length - firstArrayPosition - len) == 0) {
- // We are done with the first array.
- encodedArrays.remove(0);
- firstArrayPosition = 0;
- } else {
- firstArrayPosition = firstArrayPosition + len;
- }
-
- return x;
- }
-
- /**
- * Removes INFINITY item from its internal encoded byte array store if present. Returns whether
- * INFINITY was present.
- *
- * @see #writeInfinity()
- */
- public boolean readInfinity() {
- if ((encodedArrays == null)
- || encodedArrays.isEmpty()
- || (encodedArrays.get(0).length - firstArrayPosition < 1)) {
- throw new IllegalArgumentException("Invalid encoded byte array");
- }
- byte[] store = encodedArrays.get(0);
- if (store.length - firstArrayPosition < 2) {
- return false;
- }
- if ((store[firstArrayPosition] == ESCAPE2) && (store[firstArrayPosition + 1] == INFINITY)) {
- if ((store.length - firstArrayPosition - 2) == 0) {
- // we are done with the first array
- encodedArrays.remove(0);
- firstArrayPosition = 0;
- } else {
- firstArrayPosition = firstArrayPosition + 2;
- }
- return true;
- } else {
- return false;
- }
- }
-
- /**
- * Returns the trailing byte array item from its internal encoded byte array store and removes the
- * item from the store.
- *
- * @see #writeTrailingBytes(byte[])
- */
- public byte[] readTrailingBytes() {
- // one item is contained within one byte array
- if ((encodedArrays == null) || (encodedArrays.size() != 1)) {
- throw new IllegalArgumentException("Invalid encoded byte array");
- }
-
- byte[] store = encodedArrays.get(0);
- encodedArrays.remove(0);
- assert encodedArrays.isEmpty();
- return Arrays.copyOfRange(store, firstArrayPosition, store.length);
- }
-
- /**
- * Reads (unencoded) {@code len} bytes.
- *
- * @see #writeTrailingBytes(byte[])
- */
- public byte[] readBytes(int len) {
- if ((encodedArrays == null)
- || encodedArrays.isEmpty()
- || (encodedArrays.get(0).length - firstArrayPosition < len)) {
- throw new IllegalArgumentException("Invalid encoded byte array");
- }
-
- byte[] store = encodedArrays.get(0);
-
- byte[] result;
- if (store.length - firstArrayPosition == len) {
- // We are done with the first array.
- result = encodedArrays.remove(0);
- firstArrayPosition = 0;
- } else {
- result = new byte[len];
- System.arraycopy(store, firstArrayPosition, result, 0, len);
- firstArrayPosition = firstArrayPosition + len;
- }
- return result;
- }
-
- /**
- * Returns the encoded bytes that represent the current state of the OrderedCode.
- *
- * NOTE: This method returns OrderedCode's internal array (not a copy) for better
- * performance. Therefore the returned array should not be modified.
- */
- public byte[] getEncodedBytes() {
- if (encodedArrays.isEmpty()) {
- return new byte[0];
- }
- if ((encodedArrays.size() == 1) && (firstArrayPosition == 0)) {
- return encodedArrays.get(0);
- }
-
- int totalLength = 0;
-
- for (int i = 0; i < encodedArrays.size(); i++) {
- byte[] bytes = encodedArrays.get(i);
- if (i == 0) {
- totalLength += bytes.length - firstArrayPosition;
- } else {
- totalLength += bytes.length;
- }
- }
-
- byte[] encodedBytes = new byte[totalLength];
- int destPos = 0;
- for (int i = 0; i < encodedArrays.size(); i++) {
- byte[] bytes = encodedArrays.get(i);
- if (i == 0) {
- System.arraycopy(
- bytes, firstArrayPosition, encodedBytes, destPos, bytes.length - firstArrayPosition);
- destPos += bytes.length - firstArrayPosition;
- } else {
- System.arraycopy(bytes, 0, encodedBytes, destPos, bytes.length);
- destPos += bytes.length;
- }
- }
-
- // replace the store with merged array, so that repeated calls
- // don't need to merge. The reads can handle both the versions.
- encodedArrays.clear();
- encodedArrays.add(encodedBytes);
- firstArrayPosition = 0;
-
- return encodedBytes;
- }
-
- /**
- * Returns true if this has more encoded bytes that haven't been read, false otherwise. Return
- * value of true doesn't imply anything about validity of remaining data.
- *
- * @return true if it has more encoded bytes that haven't been read, false otherwise.
- */
- public boolean hasRemainingEncodedBytes() {
- // We delete an array after fully consuming it.
- return encodedArrays != null && encodedArrays.size() != 0;
- }
-}
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/OrderedCodeTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/OrderedCodeTest.java
deleted file mode 100644
index 58e2413e887..00000000000
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/OrderedCodeTest.java
+++ /dev/null
@@ -1,489 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.dataflow.worker;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding;
-import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Bytes;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for OrderedCode. */
-@RunWith(JUnit4.class)
-public class OrderedCodeTest {
- @Test
- public void testWriteInfinity() {
- OrderedCode orderedCode = new OrderedCode();
- try {
- orderedCode.readInfinity();
- fail("Expected IllegalArgumentException.");
- } catch (IllegalArgumentException e) {
- // expected
- }
- orderedCode.writeInfinity();
- assertTrue(orderedCode.readInfinity());
- try {
- orderedCode.readInfinity();
- fail("Expected IllegalArgumentException.");
- } catch (IllegalArgumentException e) {
- // expected
- }
- }
-
- @Test
- public void testWriteBytes() {
- byte[] first = {'a', 'b', 'c'};
- byte[] second = {'d', 'e', 'f'};
- byte[] last = {'x', 'y', 'z'};
- OrderedCode orderedCode = new OrderedCode();
- orderedCode.writeBytes(first);
- byte[] firstEncoded = orderedCode.getEncodedBytes();
- assertArrayEquals(orderedCode.readBytes(), first);
-
- orderedCode.writeBytes(first);
- orderedCode.writeBytes(second);
- orderedCode.writeBytes(last);
- byte[] allEncoded = orderedCode.getEncodedBytes();
- assertArrayEquals(orderedCode.readBytes(), first);
- assertArrayEquals(orderedCode.readBytes(), second);
- assertArrayEquals(orderedCode.readBytes(), last);
-
- orderedCode = new OrderedCode(firstEncoded);
- orderedCode.writeBytes(second);
- orderedCode.writeBytes(last);
- assertArrayEquals(orderedCode.getEncodedBytes(), allEncoded);
- assertArrayEquals(orderedCode.readBytes(), first);
- assertArrayEquals(orderedCode.readBytes(), second);
- assertArrayEquals(orderedCode.readBytes(), last);
-
- orderedCode = new OrderedCode(allEncoded);
- assertArrayEquals(orderedCode.readBytes(), first);
- assertArrayEquals(orderedCode.readBytes(), second);
- assertArrayEquals(orderedCode.readBytes(), last);
- }
-
- @Test
- public void testWriteNumIncreasing() {
- OrderedCode orderedCode = new OrderedCode();
- orderedCode.writeNumIncreasing(0);
- orderedCode.writeNumIncreasing(1);
- orderedCode.writeNumIncreasing(Long.MIN_VALUE);
- orderedCode.writeNumIncreasing(Long.MAX_VALUE);
- assertEquals(0, orderedCode.readNumIncreasing());
- assertEquals(1, orderedCode.readNumIncreasing());
- assertEquals(Long.MIN_VALUE, orderedCode.readNumIncreasing());
- assertEquals(Long.MAX_VALUE, orderedCode.readNumIncreasing());
- }
-
- /**
- * Assert that encoding the specified long via {@link OrderedCode#writeSignedNumIncreasing(long)}
- * results in the bytes represented by the specified string of hex digits. E.g.
- * assertSignedNumIncreasingEncodingEquals("3fbf", -65) asserts that -65 is encoded as { (byte)
- * 0x3f, (byte) 0xbf }.
- */
- private static void assertSignedNumIncreasingEncodingEquals(
- String expectedHexEncoding, long num) {
- OrderedCode orderedCode = new OrderedCode();
- orderedCode.writeSignedNumIncreasing(num);
- assertEquals(
- "Unexpected encoding for " + num,
- expectedHexEncoding,
- BaseEncoding.base16().lowerCase().encode(orderedCode.getEncodedBytes()));
- }
-
- /**
- * Assert that encoding various long values via {@link OrderedCode#writeSignedNumIncreasing(long)}
- * produces the expected bytes. Expected byte sequences were generated via the c++ (authoritative)
- * implementation of OrderedCode::WriteSignedNumIncreasing.
- */
- @Test
- public void testSignedNumIncreasing_write() {
- assertSignedNumIncreasingEncodingEquals("003f8000000000000000", Long.MIN_VALUE);
- assertSignedNumIncreasingEncodingEquals("003f8000000000000001", Long.MIN_VALUE + 1);
- assertSignedNumIncreasingEncodingEquals("077fffffff", Integer.MIN_VALUE - 1L);
- assertSignedNumIncreasingEncodingEquals("0780000000", Integer.MIN_VALUE);
- assertSignedNumIncreasingEncodingEquals("0780000001", Integer.MIN_VALUE + 1);
- assertSignedNumIncreasingEncodingEquals("3fbf", -65);
- assertSignedNumIncreasingEncodingEquals("40", -64);
- assertSignedNumIncreasingEncodingEquals("41", -63);
- assertSignedNumIncreasingEncodingEquals("7d", -3);
- assertSignedNumIncreasingEncodingEquals("7e", -2);
- assertSignedNumIncreasingEncodingEquals("7f", -1);
- assertSignedNumIncreasingEncodingEquals("80", 0);
- assertSignedNumIncreasingEncodingEquals("81", 1);
- assertSignedNumIncreasingEncodingEquals("82", 2);
- assertSignedNumIncreasingEncodingEquals("83", 3);
- assertSignedNumIncreasingEncodingEquals("bf", 63);
- assertSignedNumIncreasingEncodingEquals("c040", 64);
- assertSignedNumIncreasingEncodingEquals("c041", 65);
- assertSignedNumIncreasingEncodingEquals("f87ffffffe", Integer.MAX_VALUE - 1);
- assertSignedNumIncreasingEncodingEquals("f87fffffff", Integer.MAX_VALUE);
- assertSignedNumIncreasingEncodingEquals("f880000000", Integer.MAX_VALUE + 1L);
- assertSignedNumIncreasingEncodingEquals("ffc07ffffffffffffffe", Long.MAX_VALUE - 1);
- assertSignedNumIncreasingEncodingEquals("ffc07fffffffffffffff", Long.MAX_VALUE);
- }
-
- /**
- * Convert a string of hex digits (e.g. "3fbf") to a byte[] (e.g. { (byte) 0x3f, (byte) 0xbf }).
- */
- private static byte[] bytesFromHexString(String hexDigits) {
- return BaseEncoding.base16().lowerCase().decode(hexDigits);
- }
-
- /**
- * Assert that decoding (via {@link OrderedCode#readSignedNumIncreasing()}) the bytes represented
- * by the specified string of hex digits results in the expected long value. E.g.
- * assertDecodedSignedNumIncreasingEquals(-65, "3fbf") asserts that the byte array { (byte) 0x3f,
- * (byte) 0xbf } is decoded as -65.
- */
- private static void assertDecodedSignedNumIncreasingEquals(
- long expectedNum, String encodedHexString) {
- OrderedCode orderedCode = new OrderedCode(bytesFromHexString(encodedHexString));
- assertEquals(
- "Unexpected value when decoding 0x" + encodedHexString,
- expectedNum,
- orderedCode.readSignedNumIncreasing());
- assertFalse(
- "Unexpected encoded bytes remain after decoding 0x" + encodedHexString,
- orderedCode.hasRemainingEncodedBytes());
- }
-
- /**
- * Assert that decoding various sequences of bytes via {@link
- * OrderedCode#readSignedNumIncreasing()} produces the expected long value. Input byte sequences
- * were generated via the c++ (authoritative) implementation of
- * OrderedCode::WriteSignedNumIncreasing.
- */
- @Test
- public void testSignedNumIncreasing_read() {
- assertDecodedSignedNumIncreasingEquals(Long.MIN_VALUE, "003f8000000000000000");
- assertDecodedSignedNumIncreasingEquals(Long.MIN_VALUE + 1, "003f8000000000000001");
- assertDecodedSignedNumIncreasingEquals(Integer.MIN_VALUE - 1L, "077fffffff");
- assertDecodedSignedNumIncreasingEquals(Integer.MIN_VALUE, "0780000000");
- assertDecodedSignedNumIncreasingEquals(Integer.MIN_VALUE + 1, "0780000001");
- assertDecodedSignedNumIncreasingEquals(-65, "3fbf");
- assertDecodedSignedNumIncreasingEquals(-64, "40");
- assertDecodedSignedNumIncreasingEquals(-63, "41");
- assertDecodedSignedNumIncreasingEquals(-3, "7d");
- assertDecodedSignedNumIncreasingEquals(-2, "7e");
- assertDecodedSignedNumIncreasingEquals(-1, "7f");
- assertDecodedSignedNumIncreasingEquals(0, "80");
- assertDecodedSignedNumIncreasingEquals(1, "81");
- assertDecodedSignedNumIncreasingEquals(2, "82");
- assertDecodedSignedNumIncreasingEquals(3, "83");
- assertDecodedSignedNumIncreasingEquals(63, "bf");
- assertDecodedSignedNumIncreasingEquals(64, "c040");
- assertDecodedSignedNumIncreasingEquals(65, "c041");
- assertDecodedSignedNumIncreasingEquals(Integer.MAX_VALUE - 1, "f87ffffffe");
- assertDecodedSignedNumIncreasingEquals(Integer.MAX_VALUE, "f87fffffff");
- assertDecodedSignedNumIncreasingEquals(Integer.MAX_VALUE + 1L, "f880000000");
- assertDecodedSignedNumIncreasingEquals(Long.MAX_VALUE - 1, "ffc07ffffffffffffffe");
- assertDecodedSignedNumIncreasingEquals(Long.MAX_VALUE, "ffc07fffffffffffffff");
- }
-
- /**
- * Assert that encoding (via {@link OrderedCode#writeSignedNumIncreasing(long)}) the specified
- * long value and then decoding (via {@link OrderedCode#readSignedNumIncreasing()}) results in the
- * original value.
- */
- private static void assertSignedNumIncreasingWriteAndReadIsLossless(long num) {
- OrderedCode orderedCode = new OrderedCode();
- orderedCode.writeSignedNumIncreasing(num);
- assertEquals(
- "Unexpected result when decoding writeSignedNumIncreasing(" + num + ")",
- num,
- orderedCode.readSignedNumIncreasing());
- assertFalse(
- "Unexpected remaining encoded bytes after decoding " + num,
- orderedCode.hasRemainingEncodedBytes());
- }
-
- /**
- * Assert that for various long values, encoding (via {@link
- * OrderedCode#writeSignedNumIncreasing(long)}) and then decoding (via {@link
- * OrderedCode#readSignedNumIncreasing()}) results in the original value.
- */
- @Test
- public void testSignedNumIncreasing_writeAndRead() {
- assertSignedNumIncreasingWriteAndReadIsLossless(Long.MIN_VALUE);
- assertSignedNumIncreasingWriteAndReadIsLossless(Long.MIN_VALUE + 1);
- assertSignedNumIncreasingWriteAndReadIsLossless(Integer.MIN_VALUE - 1L);
- assertSignedNumIncreasingWriteAndReadIsLossless(Integer.MIN_VALUE);
- assertSignedNumIncreasingWriteAndReadIsLossless(Integer.MIN_VALUE + 1);
- assertSignedNumIncreasingWriteAndReadIsLossless(-65);
- assertSignedNumIncreasingWriteAndReadIsLossless(-64);
- assertSignedNumIncreasingWriteAndReadIsLossless(-63);
- assertSignedNumIncreasingWriteAndReadIsLossless(-3);
- assertSignedNumIncreasingWriteAndReadIsLossless(-2);
- assertSignedNumIncreasingWriteAndReadIsLossless(-1);
- assertSignedNumIncreasingWriteAndReadIsLossless(0);
- assertSignedNumIncreasingWriteAndReadIsLossless(1);
- assertSignedNumIncreasingWriteAndReadIsLossless(2);
- assertSignedNumIncreasingWriteAndReadIsLossless(3);
- assertSignedNumIncreasingWriteAndReadIsLossless(63);
- assertSignedNumIncreasingWriteAndReadIsLossless(64);
- assertSignedNumIncreasingWriteAndReadIsLossless(65);
- assertSignedNumIncreasingWriteAndReadIsLossless(Integer.MAX_VALUE - 1);
- assertSignedNumIncreasingWriteAndReadIsLossless(Integer.MAX_VALUE);
- assertSignedNumIncreasingWriteAndReadIsLossless(Integer.MAX_VALUE + 1L);
- assertSignedNumIncreasingWriteAndReadIsLossless(Long.MAX_VALUE - 1);
- assertSignedNumIncreasingWriteAndReadIsLossless(Long.MAX_VALUE);
- }
-
- @Test
- public void testLog2Floor_Positive() {
- OrderedCode orderedCode = new OrderedCode();
- assertEquals(0, orderedCode.log2Floor(1));
- assertEquals(1, orderedCode.log2Floor(2));
- assertEquals(1, orderedCode.log2Floor(3));
- assertEquals(2, orderedCode.log2Floor(4));
- assertEquals(5, orderedCode.log2Floor(63));
- assertEquals(6, orderedCode.log2Floor(64));
- assertEquals(62, orderedCode.log2Floor(Long.MAX_VALUE));
- }
-
- /** OrderedCode.log2Floor(long) is defined to return -1 given an input of zero. */
- @Test
- public void testLog2Floor_zero() {
- OrderedCode orderedCode = new OrderedCode();
- assertEquals(-1, orderedCode.log2Floor(0));
- }
-
- @Test
- public void testLog2Floor_negative() {
- OrderedCode orderedCode = new OrderedCode();
- try {
- orderedCode.log2Floor(-1);
- fail("Expected an IllegalArgumentException.");
- } catch (IllegalArgumentException expected) {
- // Expected!
- }
- }
-
- @Test
- public void testGetSignedEncodingLength() {
- OrderedCode orderedCode = new OrderedCode();
- assertEquals(10, orderedCode.getSignedEncodingLength(Long.MIN_VALUE));
- assertEquals(10, orderedCode.getSignedEncodingLength(~(1L << 62)));
- assertEquals(9, orderedCode.getSignedEncodingLength(~(1L << 62) + 1));
- assertEquals(3, orderedCode.getSignedEncodingLength(-8193));
- assertEquals(2, orderedCode.getSignedEncodingLength(-8192));
- assertEquals(2, orderedCode.getSignedEncodingLength(-65));
- assertEquals(1, orderedCode.getSignedEncodingLength(-64));
- assertEquals(1, orderedCode.getSignedEncodingLength(-2));
- assertEquals(1, orderedCode.getSignedEncodingLength(-1));
- assertEquals(1, orderedCode.getSignedEncodingLength(0));
- assertEquals(1, orderedCode.getSignedEncodingLength(1));
- assertEquals(1, orderedCode.getSignedEncodingLength(63));
- assertEquals(2, orderedCode.getSignedEncodingLength(64));
- assertEquals(2, orderedCode.getSignedEncodingLength(8191));
- assertEquals(3, orderedCode.getSignedEncodingLength(8192));
- assertEquals(9, orderedCode.getSignedEncodingLength((1L << 62)) - 1);
- assertEquals(10, orderedCode.getSignedEncodingLength(1L << 62));
- assertEquals(10, orderedCode.getSignedEncodingLength(Long.MAX_VALUE));
- }
-
- @Test
- public void testWriteTrailingBytes() {
- byte[] escapeChars =
- new byte[] {
- OrderedCode.ESCAPE1,
- OrderedCode.NULL_CHARACTER,
- OrderedCode.SEPARATOR,
- OrderedCode.ESCAPE2,
- OrderedCode.INFINITY,
- OrderedCode.FF_CHARACTER
- };
- byte[] anotherArray = new byte[] {'a', 'b', 'c', 'd', 'e'};
-
- OrderedCode orderedCode = new OrderedCode();
- orderedCode.writeTrailingBytes(escapeChars);
- assertArrayEquals(orderedCode.getEncodedBytes(), escapeChars);
- assertArrayEquals(orderedCode.readTrailingBytes(), escapeChars);
- try {
- orderedCode.readInfinity();
- fail("Expected IllegalArgumentException.");
- } catch (IllegalArgumentException e) {
- // expected
- }
-
- orderedCode = new OrderedCode();
- orderedCode.writeTrailingBytes(anotherArray);
- assertArrayEquals(orderedCode.getEncodedBytes(), anotherArray);
- assertArrayEquals(orderedCode.readTrailingBytes(), anotherArray);
- }
-
- @Test
- public void testMixedWrite() {
- byte[] first = {'a', 'b', 'c'};
- byte[] second = {'d', 'e', 'f'};
- byte[] last = {'x', 'y', 'z'};
- byte[] escapeChars =
- new byte[] {
- OrderedCode.ESCAPE1,
- OrderedCode.NULL_CHARACTER,
- OrderedCode.SEPARATOR,
- OrderedCode.ESCAPE2,
- OrderedCode.INFINITY,
- OrderedCode.FF_CHARACTER
- };
-
- OrderedCode orderedCode = new OrderedCode();
- orderedCode.writeBytes(first);
- orderedCode.writeBytes(second);
- orderedCode.writeBytes(last);
- orderedCode.writeInfinity();
- orderedCode.writeNumIncreasing(0);
- orderedCode.writeNumIncreasing(1);
- orderedCode.writeNumIncreasing(Long.MIN_VALUE);
- orderedCode.writeNumIncreasing(Long.MAX_VALUE);
- orderedCode.writeSignedNumIncreasing(0);
- orderedCode.writeSignedNumIncreasing(1);
- orderedCode.writeSignedNumIncreasing(Long.MIN_VALUE);
- orderedCode.writeSignedNumIncreasing(Long.MAX_VALUE);
- orderedCode.writeTrailingBytes(escapeChars);
- byte[] allEncoded = orderedCode.getEncodedBytes();
- assertArrayEquals(orderedCode.readBytes(), first);
- assertArrayEquals(orderedCode.readBytes(), second);
- assertFalse(orderedCode.readInfinity());
- assertArrayEquals(orderedCode.readBytes(), last);
- assertTrue(orderedCode.readInfinity());
- assertEquals(0, orderedCode.readNumIncreasing());
- assertEquals(1, orderedCode.readNumIncreasing());
- assertFalse(orderedCode.readInfinity());
- assertEquals(Long.MIN_VALUE, orderedCode.readNumIncreasing());
- assertEquals(Long.MAX_VALUE, orderedCode.readNumIncreasing());
- assertEquals(0, orderedCode.readSignedNumIncreasing());
- assertEquals(1, orderedCode.readSignedNumIncreasing());
- assertFalse(orderedCode.readInfinity());
- assertEquals(Long.MIN_VALUE, orderedCode.readSignedNumIncreasing());
- assertEquals(Long.MAX_VALUE, orderedCode.readSignedNumIncreasing());
- assertArrayEquals(orderedCode.getEncodedBytes(), escapeChars);
- assertArrayEquals(orderedCode.readTrailingBytes(), escapeChars);
-
- orderedCode = new OrderedCode(allEncoded);
- assertArrayEquals(orderedCode.readBytes(), first);
- assertArrayEquals(orderedCode.readBytes(), second);
- assertFalse(orderedCode.readInfinity());
- assertArrayEquals(orderedCode.readBytes(), last);
- assertTrue(orderedCode.readInfinity());
- assertEquals(0, orderedCode.readNumIncreasing());
- assertEquals(1, orderedCode.readNumIncreasing());
- assertFalse(orderedCode.readInfinity());
- assertEquals(Long.MIN_VALUE, orderedCode.readNumIncreasing());
- assertEquals(Long.MAX_VALUE, orderedCode.readNumIncreasing());
- assertEquals(0, orderedCode.readSignedNumIncreasing());
- assertEquals(1, orderedCode.readSignedNumIncreasing());
- assertFalse(orderedCode.readInfinity());
- assertEquals(Long.MIN_VALUE, orderedCode.readSignedNumIncreasing());
- assertEquals(Long.MAX_VALUE, orderedCode.readSignedNumIncreasing());
- assertArrayEquals(orderedCode.getEncodedBytes(), escapeChars);
- assertArrayEquals(orderedCode.readTrailingBytes(), escapeChars);
- }
-
- @Test
- public void testEdgeCases() {
- byte[] ffChar = {OrderedCode.FF_CHARACTER};
- byte[] nullChar = {OrderedCode.NULL_CHARACTER};
-
- byte[] separatorEncoded = {OrderedCode.ESCAPE1, OrderedCode.SEPARATOR};
- byte[] ffCharEncoded = {OrderedCode.ESCAPE1, OrderedCode.NULL_CHARACTER};
- byte[] nullCharEncoded = {OrderedCode.ESCAPE2, OrderedCode.FF_CHARACTER};
- byte[] infinityEncoded = {OrderedCode.ESCAPE2, OrderedCode.INFINITY};
-
- OrderedCode orderedCode = new OrderedCode();
- orderedCode.writeBytes(ffChar);
- orderedCode.writeBytes(nullChar);
- orderedCode.writeInfinity();
- assertArrayEquals(
- orderedCode.getEncodedBytes(),
- Bytes.concat(
- ffCharEncoded, separatorEncoded, nullCharEncoded, separatorEncoded, infinityEncoded));
- assertArrayEquals(orderedCode.readBytes(), ffChar);
- assertArrayEquals(orderedCode.readBytes(), nullChar);
- assertTrue(orderedCode.readInfinity());
-
- orderedCode = new OrderedCode(Bytes.concat(ffCharEncoded, separatorEncoded));
- assertArrayEquals(orderedCode.readBytes(), ffChar);
-
- orderedCode = new OrderedCode(Bytes.concat(nullCharEncoded, separatorEncoded));
- assertArrayEquals(orderedCode.readBytes(), nullChar);
-
- byte[] invalidEncodingForRead = {
- OrderedCode.ESCAPE2, OrderedCode.ESCAPE2, OrderedCode.ESCAPE1, OrderedCode.SEPARATOR
- };
- orderedCode = new OrderedCode(invalidEncodingForRead);
- try {
- orderedCode.readBytes();
- fail("Should have failed.");
- } catch (Exception e) {
- // Expected
- }
- assertTrue(orderedCode.hasRemainingEncodedBytes());
- }
-
- @Test
- public void testHasRemainingEncodedBytes() {
- byte[] bytes = {'a', 'b', 'c'};
- long number = 12345;
-
- // Empty
- OrderedCode orderedCode = new OrderedCode();
- assertFalse(orderedCode.hasRemainingEncodedBytes());
-
- // First and only field of each type.
- orderedCode.writeBytes(bytes);
- assertTrue(orderedCode.hasRemainingEncodedBytes());
- assertArrayEquals(orderedCode.readBytes(), bytes);
- assertFalse(orderedCode.hasRemainingEncodedBytes());
-
- orderedCode.writeNumIncreasing(number);
- assertTrue(orderedCode.hasRemainingEncodedBytes());
- assertEquals(orderedCode.readNumIncreasing(), number);
- assertFalse(orderedCode.hasRemainingEncodedBytes());
-
- orderedCode.writeSignedNumIncreasing(number);
- assertTrue(orderedCode.hasRemainingEncodedBytes());
- assertEquals(orderedCode.readSignedNumIncreasing(), number);
- assertFalse(orderedCode.hasRemainingEncodedBytes());
-
- orderedCode.writeInfinity();
- assertTrue(orderedCode.hasRemainingEncodedBytes());
- assertTrue(orderedCode.readInfinity());
- assertFalse(orderedCode.hasRemainingEncodedBytes());
-
- orderedCode.writeTrailingBytes(bytes);
- assertTrue(orderedCode.hasRemainingEncodedBytes());
- assertArrayEquals(orderedCode.readTrailingBytes(), bytes);
- assertFalse(orderedCode.hasRemainingEncodedBytes());
-
- // Two fields of same type.
- orderedCode.writeBytes(bytes);
- orderedCode.writeBytes(bytes);
- assertTrue(orderedCode.hasRemainingEncodedBytes());
- assertArrayEquals(orderedCode.readBytes(), bytes);
- assertArrayEquals(orderedCode.readBytes(), bytes);
- assertFalse(orderedCode.hasRemainingEncodedBytes());
- }
-}