Merge branch 'main' into snyk-upgrade-dafe6fd4d1eb795d6504c32641f4b46f

datastax · Dec 12, 2023 · a5ff3c2 · a5ff3c2
2 parents a876016 + b9b5e39
commit a5ff3c2
Show file tree

Hide file tree

Showing 130 changed files with 1,923 additions and 100 deletions.
diff --git a/.all-contributorsrc b/.all-contributorsrc
@@ -139,6 +139,16 @@
       "contributions": [
         "review"
       ]
+    },
+    {
+      "login": "Jeremya",
+      "name": "Jeremy",
+      "avatar_url": "https://avatars.githubusercontent.com/u/576519?v=4",
+      "profile": "https://github.com/Jeremya",
+      "contributions": [
+        "code"
+      ]
     }
-  ]
+  ],
+  "commitType": "docs"
 }
diff --git a/.github/workflows/cdm-integrationtest.yml b/.github/workflows/cdm-integrationtest.yml
@@ -1,14 +1,28 @@
 name: Build and test jar with integration tests
-on: [push]
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  cancel-in-progress: true
+
 jobs:
   CDM-Integration-Test:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        jdk: [ 8 ]
+        os: [ ubuntu-latest ]
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK 8
         uses: actions/setup-java@v3
         with:
-          java-version: '8'
+          java-version: ${{ matrix.jdk }}
           distribution: 'temurin'
           cache: maven
       - name: Test SIT with cdm

diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
@@ -8,19 +8,32 @@
 
 name: Java CI with Maven
 
-on: [push, pull_request]
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  cancel-in-progress: true
 
 jobs:
   build:
+    strategy:
+      matrix:
+        jdk: [ 8 ]
+        os: [ ubuntu-latest ]
 
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v3
     - name: Set up JDK 8
       uses: actions/setup-java@v3
       with:
-        java-version: '8'
+        java-version: ${{ matrix.jdk }}
         distribution: 'temurin'
         cache: maven
     - name: Build with Maven

diff --git a/.github/workflows/snyk-cli-scan.yml b/.github/workflows/snyk-cli-scan.yml
@@ -10,5 +10,11 @@ on:
     branches: [ main ]
   workflow_dispatch:
 
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  #group: ${{ github.workflow }}-${{ github.ref }}-${{ github.job || github.run_id }}
+  cancel-in-progress: true
+
 env:
   SNYK_SEVERITY_THRESHOLD_LEVEL: critical
+
diff --git a/.github/workflows/snyk-pr-cleanup.yml b/.github/workflows/snyk-pr-cleanup.yml
@@ -9,3 +9,8 @@ on:
       - main
   workflow_dispatch:
 
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  #group: ${{ github.workflow }}-${{ github.ref }}-${{ github.job || github.run_id }}
+  cancel-in-progress: true
+
diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs
diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs
diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -149,6 +149,7 @@ For recognizing contributions, please follow [this documentation](https://allcon
       <td align="center" valign="top" width="16.66%"><a href="https://github.com/vaishakhbn"><img src="https://avatars.githubusercontent.com/u/2619002?v=4?s=50" width="50px;" alt="Vaishakh Baragur Narasimhareddy"/><br /><sub><b>Vaishakh Baragur Narasimhareddy</b></sub></a><br /><a href="https://github.com/datastax/cassandra-data-migrator/commits?author=vaishakhbn" title="Code">💻</a> <a href="https://github.com/datastax/cassandra-data-migrator/commits?author=vaishakhbn" title="Tests">⚠️</a></td>
       <td align="center" valign="top" width="16.66%"><a href="https://github.com/mieslep"><img src="https://avatars.githubusercontent.com/u/5420540?v=4?s=50" width="50px;" alt="Phil Miesle"/><br /><sub><b>Phil Miesle</b></sub></a><br /><a href="https://github.com/datastax/cassandra-data-migrator/commits?author=mieslep" title="Code">💻</a></td>
       <td align="center" valign="top" width="16.66%"><a href="https://github.com/mfmaher2"><img src="https://avatars.githubusercontent.com/u/64795956?v=4?s=50" width="50px;" alt="mfmaher2"/><br /><sub><b>mfmaher2</b></sub></a><br /><a href="https://github.com/datastax/cassandra-data-migrator/pulls?q=is%3Apr+reviewed-by%3Amfmaher2" title="Reviewed Pull Requests">👀</a></td>
+      <td align="center" valign="top" width="16.66%"><a href="https://github.com/Jeremya"><img src="https://avatars.githubusercontent.com/u/576519?v=4?s=50" width="50px;" alt="Jeremy"/><br /><sub><b>Jeremy</b></sub></a><br /><a href="https://github.com/datastax/cassandra-data-migrator/commits?author=Jeremya" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

diff --git a/README.md b/README.md
@@ -18,12 +18,17 @@ Migrate and Validate Tables between Origin and Target Cassandra Clusters.
 
 ### Prerequisite
 - Install Java8 as spark binaries are compiled with it.
-- Install Spark version [3.4.1](https://archive.apache.org/dist/spark/spark-3.4.1/) on a single VM (no cluster necessary) where you want to run this job. Spark can be installed by running the following: -
+- Install Spark version [3.4.1](https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3-scala2.13.tgz) on a single VM (no cluster necessary) where you want to run this job. Spark can be installed by running the following: -
 ```
 wget https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3-scala2.13.tgz
 tar -xvzf spark-3.4.1-bin-hadoop3-scala2.13.tgz
 ```
 
+> :warning: If the above Spark and Scala version is not properly installed, you'll then see a similar exception like below when running the CDM jobs,
+```
+Exception in thread "main" java.lang.NoSuchMethodError: scala.runtime.Statics.releaseFence()V
+```
+
 # Steps for Data-Migration:
 
 > :warning: Note that Version 4 of the tool is not backward-compatible with .properties files created in previous versions, and that package names have changed.
@@ -35,9 +40,9 @@ tar -xvzf spark-3.4.1-bin-hadoop3-scala2.13.tgz
 3. Run the below job using `spark-submit` command as shown below:
 
 ```
-./spark-submit --properties-file cdm.properties /
---conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" /
---master "local[*]" --driver-memory 25G --executor-memory 25G /
+./spark-submit --properties-file cdm.properties \
+--conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" \
+--master "local[*]" --driver-memory 25G --executor-memory 25G \
 --class com.datastax.cdm.job.Migrate cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
 ```
 
@@ -50,9 +55,9 @@ Note:
 - To run the job in Data validation mode, use class option `--class com.datastax.cdm.job.DiffData` as shown below
 
 ```
-./spark-submit --properties-file cdm.properties /
---conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" /
---master "local[*]" --driver-memory 25G --executor-memory 25G /
+./spark-submit --properties-file cdm.properties \
+--conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" \
+--master "local[*]" --driver-memory 25G --executor-memory 25G \
 --class com.datastax.cdm.job.DiffData cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
 ```
 
@@ -89,10 +94,10 @@ Note:
 Each line above represents a partition-range (`min,max`). Alternatively, you can also pass the partition-file via command-line param as shown below
 
 ```
-./spark-submit --properties-file cdm.properties /
- --conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" /
- --conf spark.cdm.tokenRange.partitionFile="/<path-to-file>/<csv-input-filename>" /
---master "local[*]" --driver-memory 25G --executor-memory 25G /
+./spark-submit --properties-file cdm.properties \
+ --conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" \
+ --conf spark.cdm.tokenRange.partitionFile="/<path-to-file>/<csv-input-filename>" \
+ --master "local[*]" --driver-memory 25G --executor-memory 25G \
  --class com.datastax.cdm.job.<Migrate|DiffData> cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
 ```
 This mode is specifically useful to processes a subset of partition-ranges that may have failed during a previous run.
@@ -103,10 +108,10 @@ This mode is specifically useful to processes a subset of partition-ranges that
 # Perform large-field Guardrail violation checks
 - The tool can be used to identify large fields from a table that may break you cluster guardrails (e.g. AstraDB has a 10MB limit for a single large field)  `--class com.datastax.cdm.job.GuardrailCheck` as shown below
 ```
-./spark-submit --properties-file cdm.properties /
---conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" /
---conf spark.cdm.feature.guardrail.colSizeInKB=10000 /
---master "local[*]" --driver-memory 25G --executor-memory 25G /
+./spark-submit --properties-file cdm.properties \
+--conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" \
+--conf spark.cdm.feature.guardrail.colSizeInKB=10000 \
+--master "local[*]" --driver-memory 25G --executor-memory 25G \
 --class com.datastax.cdm.job.GuardrailCheck cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
 ```
 

diff --git a/RELEASE.md b/RELEASE.md
@@ -1,4 +1,7 @@
 # Release Notes
+## [4.1.9 to 4.1.11] - 2023-12-11
+- Code test & coverage changes
+
 ## [4.1.8] - 2023-10-13
 - Upgraded to use Scala 2.13 
 

diff --git a/pom.xml b/pom.xml
@@ -3,15 +3,14 @@
 
   <groupId>datastax.cdm</groupId>
   <artifactId>cassandra-data-migrator</artifactId>
-  <version>4.1.9-SNAPSHOT</version>
+  <version>4.1.11-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <scala.version>2.13.12</scala.version>
     <scala.main.version>2.13</scala.main.version>
     <spark.version>3.4.1</spark.version>
-    <scalatest.version>3.2.17</scalatest.version>
     <connector.version>3.4.1</connector.version>
     <cassandra.version>5.0-alpha1</cassandra.version>
     <junit.version>5.9.1</junit.version>
@@ -151,12 +150,6 @@
     </dependency>
 
     <!-- Test Dependencies -->
-    <dependency>
-      <groupId>org.scalatest</groupId>
-      <artifactId>scalatest_${scala.main.version}</artifactId>
-      <version>${scalatest.version}</version>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.junit.jupiter</groupId>
       <artifactId>junit-jupiter-engine</artifactId>
@@ -200,20 +193,27 @@
     </resources>
     <plugins>
       <plugin>
-        <groupId>net.alchim31.maven</groupId>
-        <artifactId>scala-maven-plugin</artifactId>
-        <version>4.8.0</version>
-        <executions>
-          <execution>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>compile</goal>
-              <goal>testCompile</goal>
-            </goals>
+      <groupId>net.alchim31.maven</groupId>
+      <artifactId>scala-maven-plugin</artifactId>
+      <version>4.8.0</version>
+      <executions>
+        <execution>
+          <phase>process-sources</phase>
+          <goals>
+            <goal>compile</goal>
+            <goal>testCompile</goal>
+          </goals>
 
-          </execution>
-        </executions>
+        </execution>
+      </executions>
       </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>2.22.2</version>
+      </plugin>
+
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
@@ -242,35 +242,6 @@
           </execution>
         </executions>
       </plugin>
-      <!-- Instructions from http://www.scalatest.org/user_guide/using_the_scalatest_maven_plugin -->
-      <!-- disable surefire -->
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-surefire-plugin</artifactId>
-        <version>2.22.2</version>
-        <configuration>
-          <skipTests>true</skipTests>
-        </configuration>
-      </plugin>
-      <!-- enable scalatest -->
-      <plugin>
-        <groupId>org.scalatest</groupId>
-        <artifactId>scalatest-maven-plugin</artifactId>
-        <version>2.2.0</version>
-        <configuration>
-          <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
-          <junitxml>.</junitxml>
-          <filereports>WDF TestSuite.txt</filereports>
-        </configuration>
-        <executions>
-          <execution>
-            <id>test</id>
-            <goals>
-              <goal>test</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
 
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
@@ -304,35 +275,28 @@
             <id>jacoco-check</id>
             <phase>test</phase>
             <goals>
-              <goal>check</goal>
               <goal>report</goal>
+              <goal>check</goal>
             </goals>
             <configuration>
-              <excludes>
-                <!-- Excluding all the Scala classes -->
-                <exclude>com.datastax.cdm.job.*</exclude>
-              </excludes>
               <rules>
                 <rule>
                   <element>BUNDLE</element>
                   <limits>
                     <limit>
                       <counter>COMPLEXITY</counter>
                       <value>COVEREDRATIO</value>
-<!--                      <minimum>0.33</minimum>-->
-                      <minimum>0</minimum>
+                      <minimum>0.33</minimum>
                     </limit>
                     <limit>
                       <counter>INSTRUCTION</counter>
                       <value>COVEREDRATIO</value>
-<!--                      <minimum>41%</minimum>-->
-                      <minimum>0%</minimum>
+                      <minimum>45%</minimum>
                     </limit>
                     <limit>
                       <counter>LINE</counter>
                       <value>MISSEDCOUNT</value>
-<!--                      <maximum>1544</maximum>-->
-                      <maximum>3085</maximum>
+                      <maximum>1500</maximum>
                     </limit>
                   </limits>
                 </rule>
@@ -341,6 +305,22 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+        <version>0.15</version>
+        <configuration>
+          <excludesFile>rat-excludes.txt</excludesFile>
+        </configuration>
+        <executions>
+          <execution>
+            <phase>verify</phase>
+            <goals>
+              <goal>check</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 </project>