diff --git a/README.md b/README.md index f1271744..324c6a8b 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Migrate and Validate Tables between Origin and Target Cassandra Clusters. ### Prerequisite - Install Java8 as spark binaries are compiled with it. -- Install Spark version [3.4.1](https://archive.apache.org/dist/spark/spark-3.4.1/) on a single VM (no cluster necessary) where you want to run this job. Spark can be installed by running the following: - +- Install Spark version [3.4.1](https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3-scala2.13.tgz) on a single VM (no cluster necessary) where you want to run this job. Spark can be installed by running the following: - ``` wget https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3-scala2.13.tgz tar -xvzf spark-3.4.1-bin-hadoop3-scala2.13.tgz @@ -97,7 +97,7 @@ Each line above represents a partition-range (`min,max`). Alternatively, you can ./spark-submit --properties-file cdm.properties \ --conf spark.cdm.schema.origin.keyspaceTable="." \ --conf spark.cdm.tokenRange.partitionFile="//" \ ---master "local[*]" --driver-memory 25G --executor-memory 25G \ + --master "local[*]" --driver-memory 25G --executor-memory 25G \ --class com.datastax.cdm.job. cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt ``` This mode is specifically useful to processes a subset of partition-ranges that may have failed during a previous run. diff --git a/RELEASE.md b/RELEASE.md index 70376e89..f792ea4f 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,4 +1,7 @@ # Release Notes +## [4.1.9 to 4.1.11] - 2023-12-11 +- Code test & coverage changes + ## [4.1.8] - 2023-10-13 - Upgraded to use Scala 2.13 diff --git a/pom.xml b/pom.xml index a2481775..056987c5 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ datastax.cdm cassandra-data-migrator - 4.1.10-SNAPSHOT + 4.1.11-SNAPSHOT jar @@ -11,7 +11,6 @@ 2.13.12 2.13 3.4.1 - 3.2.17 3.4.1 5.0-alpha1 5.9.1 @@ -151,12 +150,6 @@ - - org.scalatest - scalatest_${scala.main.version} - ${scalatest.version} - test - org.junit.jupiter junit-jupiter-engine @@ -200,20 +193,27 @@ - net.alchim31.maven - scala-maven-plugin - 4.8.0 - - - process-sources - - compile - testCompile - + net.alchim31.maven + scala-maven-plugin + 4.8.0 + + + process-sources + + compile + testCompile + - - + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + org.apache.maven.plugins maven-shade-plugin @@ -242,35 +242,6 @@ - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 2.2.0 - - ${project.build.directory}/surefire-reports - . - WDF TestSuite.txt - - - - test - - test - - - - org.apache.maven.plugins @@ -304,14 +275,10 @@ jacoco-check test - check report + check - - - com.datastax.cdm.job.* - BUNDLE @@ -319,20 +286,17 @@ COMPLEXITY COVEREDRATIO - - 0 + 0.33 INSTRUCTION COVEREDRATIO - - 0% + 45% LINE MISSEDCOUNT - - 3085 + 1500 diff --git a/src/main/java/com/datastax/cdm/job/SplitPartitions.java b/src/main/java/com/datastax/cdm/job/SplitPartitions.java index 25d8e8df..003e892f 100644 --- a/src/main/java/com/datastax/cdm/job/SplitPartitions.java +++ b/src/main/java/com/datastax/cdm/job/SplitPartitions.java @@ -35,7 +35,7 @@ public class SplitPartitions { public static Logger logger = LoggerFactory.getLogger(SplitPartitions.class.getName()); - public static Collection getRandomSubPartitions(int numSplits, BigInteger min, BigInteger max, int coveragePercent) { + public static List getRandomSubPartitions(int numSplits, BigInteger min, BigInteger max, int coveragePercent) { logger.info("ThreadID: {} Splitting min: {} max: {}", Thread.currentThread().getId(), min, max); List partitions = getSubPartitions(numSplits, min, max, coveragePercent); Collections.shuffle(partitions); diff --git a/src/test/java/com/datastax/cdm/job/SplitPartitionsTest.java b/src/test/java/com/datastax/cdm/job/SplitPartitionsTest.java new file mode 100644 index 00000000..a4de09ce --- /dev/null +++ b/src/test/java/com/datastax/cdm/job/SplitPartitionsTest.java @@ -0,0 +1,40 @@ +package com.datastax.cdm.job; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.math.BigInteger; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class SplitPartitionsTest { + + @Test + void getRandomSubPartitionsTest() { + List partitions = SplitPartitions.getRandomSubPartitions(10, BigInteger.ONE, + BigInteger.valueOf(100), 100); + assertEquals(10, partitions.size()); + partitions.forEach(p -> { + assertEquals(9, p.getMax().longValue() - p.getMin().longValue()); + }); + } + + @Test + void getRandomSubPartitionsTestOver100() { + List partitions = SplitPartitions.getRandomSubPartitions(8, BigInteger.ONE, + BigInteger.valueOf(44), 200); + assertEquals(8, partitions.size()); + } + @Test + void batchesTest() { + List mutable_list = Arrays.asList("e1", "e2", "e3", "e4", "e5", "e6"); + Stream> out = SplitPartitions.batches(mutable_list, 2); + assertEquals(3, out.count()); + } + +}