Skip to content
This repository has been archived by the owner on Feb 12, 2022. It is now read-only.

Commit

Permalink
Rename CleanupNonStarportPipelines to CleanupUnmanagedPipelines (#53)
Browse files Browse the repository at this point in the history
  • Loading branch information
sushengloong authored and realstraw committed Dec 11, 2019
1 parent f4e7e2c commit 37f6349
Show file tree
Hide file tree
Showing 8 changed files with 26 additions and 33 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ import com.krux.starport.util.{AwsDataPipeline, PipelineState, ErrorHandler}
*/
object CleanupExistingPipelines extends StarportActivity {

val taskName = "CleanupExistingPipelines"

val metrics = new MetricRegistry()

def activePipelineRecords(): Int = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,12 @@ import org.joda.time.DateTime
import org.joda.time.format.{DateTimeFormat => JodaDateTimeFormat}
import slick.jdbc.PostgresProfile.api._

import com.krux.starport.cli.{CleanupNonStarportOptionParser, CleanupNonStarportOptions}
import com.krux.starport.cli.{CleanupUnmanagedOptionParser, CleanupUnmanagedOptions}
import com.krux.starport.db.table.ScheduledPipelines
import com.krux.starport.util.{AwsDataPipeline, PipelineStatus, PipelineState}

object CleanupNonStarportPipelines extends StarportActivity {
object CleanupUnmanagedPipelines extends StarportActivity {
final val AwsDateTimeFormat = "yyyy-MM-dd'T'HH:mm:ss"
final val taskName = "CleanupNonStarportPipelines"

def pipelineIdsToDelete(
excludePrefixes: Seq[String],
Expand All @@ -20,7 +19,7 @@ object CleanupNonStarportPipelines extends StarportActivity {
force: Boolean
): Set[String] = {

logger.info(s"Getting list of old ${pipelineState} non-Starport pipelines from AWS to delete...")
logger.info(s"Getting list of old ${pipelineState} unmanaged pipelines from AWS to delete...")
val dateTimeFormatter = JodaDateTimeFormat.forPattern(AwsDateTimeFormat)
val inConsoleStarportScheduledPipelineIds = db.run(
ScheduledPipelines()
Expand Down Expand Up @@ -69,7 +68,7 @@ object CleanupNonStarportPipelines extends StarportActivity {
}
}

def run(options: CleanupNonStarportOptions) = {
def run(options: CleanupUnmanagedOptions) = {
logger.info(s"run with options: $options")

val ids = pipelineIdsToDelete(options.excludePrefixes, options.pipelineState, options.cutoffDate, options.force)
Expand All @@ -80,11 +79,11 @@ object CleanupNonStarportPipelines extends StarportActivity {

def main(args: Array[String]): Unit = {
val start = System.nanoTime()
CleanupNonStarportOptionParser.parse(args) match {
CleanupUnmanagedOptionParser.parse(args) match {
case Some(options) => run(options)
case None => ErrorExit.invalidCommandlineArguments(logger)
}
val timeSpan = (System.nanoTime - start) / 1E9
logger.info(s"All old and FINISHED non-Starport pipelines cleaned up in $timeSpan seconds")
logger.info(s"All old and FINISHED unmanaged pipelines cleaned up in $timeSpan seconds")
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@ import com.krux.starport.util.DateTimeFunctions

trait StarportActivity extends DateTimeMapped with WaitForIt with Logging with DateTimeFunctions {

def taskName: String

// use -Dconf.resource=application.dev.conf for testing
implicit val conf = StarportSettings()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,6 @@ object StartScheduledPipelines extends StarportActivity {

val scheduleTimer = metrics.timer("timers.pipeline_scheduling_time")

val taskName = "StartScheduledPipelines"

val extraEnvs = conf.extraEnvs.toSeq

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@ import scopt.OptionParser

import com.krux.starport.util.PipelineState

object CleanupNonStarportOptionParser extends Reads {
val programName = "start-scheduled-pipelines"
object CleanupUnmanagedOptionParser extends Reads {
val programName = "cleanup-unmanaged-pipelines"

def apply(): OptionParser[CleanupNonStarportOptions] = new OptionParser[CleanupNonStarportOptions](programName) {
def apply(): OptionParser[CleanupUnmanagedOptions] = new OptionParser[CleanupUnmanagedOptions](programName) {

head(programName)
help("help").text("prints this usage text")
Expand All @@ -34,6 +34,6 @@ object CleanupNonStarportOptionParser extends Reads {
.action((_, c) => c.copy(force = true))
}

def parse(args: Array[String]): Option[CleanupNonStarportOptions] = apply().parse(args, CleanupNonStarportOptions())
def parse(args: Array[String]): Option[CleanupUnmanagedOptions] = apply().parse(args, CleanupUnmanagedOptions())

}
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import org.joda.time.DateTime

import com.krux.starport.util.PipelineState

case class CleanupNonStarportOptions(
case class CleanupUnmanagedOptions(
excludePrefixes: Seq[String] = Seq(),
pipelineState: PipelineState.State = PipelineState.FINISHED,
cutoffDate: DateTime = DateTime.now.minusMonths(2).withTimeAtStartOfDay,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,23 +6,23 @@ import org.scalatest.WordSpec
import com.krux.starport.util.PipelineState


class CleanupNonStarportOptionParserSpec extends WordSpec {
"CleanupNonStarportOptionParser" when {
class CleanupUnmanagedOptionParserSpec extends WordSpec {
"CleanupUnmanagedOptionParser" when {
"args is empty" should {
val args = Array.empty[String]

"return empty excludePrefixes" in {
val options = CleanupNonStarportOptionParser.parse(args).get
val options = CleanupUnmanagedOptionParser.parse(args).get
assert(options.excludePrefixes === Array.empty[String])
}

"return FINISHED as default pipelineState" in {
val options = CleanupNonStarportOptionParser.parse(args).get
val options = CleanupUnmanagedOptionParser.parse(args).get
assert(options.pipelineState === PipelineState.FINISHED)
}

"return 2 months ago as default cutoffDate" in {
val options = CleanupNonStarportOptionParser.parse(args).get
val options = CleanupUnmanagedOptionParser.parse(args).get
val cutoffDate = options.cutoffDate
val expectedDate = DateTime.now.minusMonths(2)

Expand All @@ -32,7 +32,7 @@ class CleanupNonStarportOptionParserSpec extends WordSpec {
}

"return false as default dryRun" in {
val options = CleanupNonStarportOptionParser.parse(args).get
val options = CleanupUnmanagedOptionParser.parse(args).get
assert(!options.dryRun)
}
}
Expand All @@ -48,17 +48,17 @@ class CleanupNonStarportOptionParserSpec extends WordSpec {
)

"parse excludePrefixes" in {
val options = CleanupNonStarportOptionParser.parse(args).get
val options = CleanupUnmanagedOptionParser.parse(args).get
assert(options.excludePrefixes === Seq("sp_"))
}

"parse pipelineState" in {
val options = CleanupNonStarportOptionParser.parse(args).get
val options = CleanupUnmanagedOptionParser.parse(args).get
assert(options.pipelineState === PipelineState.PENDING)
}

"parse cutoffDate" in {
val options = CleanupNonStarportOptionParser.parse(args).get
val options = CleanupUnmanagedOptionParser.parse(args).get
val cutoffDate = options.cutoffDate
val expectedDate = new DateTime("2017-06-30T00:00:00Z")

Expand All @@ -68,7 +68,7 @@ class CleanupNonStarportOptionParserSpec extends WordSpec {
}

"parse dryRun" in {
val options = CleanupNonStarportOptionParser.parse(args).get
val options = CleanupUnmanagedOptionParser.parse(args).get
assert(options.dryRun)
}
}
Expand All @@ -81,7 +81,7 @@ class CleanupNonStarportOptionParserSpec extends WordSpec {
)

"parse multiple excludePrefixes" in {
val options = CleanupNonStarportOptionParser.parse(args).get
val options = CleanupUnmanagedOptionParser.parse(args).get
assert(options.excludePrefixes === Seq("sp_env1_", "sp_env2_"))
}
}
Expand All @@ -92,7 +92,7 @@ class CleanupNonStarportOptionParserSpec extends WordSpec {
)

"return None" in {
assert(CleanupNonStarportOptionParser.parse(args) === None)
assert(CleanupUnmanagedOptionParser.parse(args) === None)
}
}
"pipelineState is not valid" should {
Expand All @@ -102,7 +102,7 @@ class CleanupNonStarportOptionParserSpec extends WordSpec {
)

"return None" in {
assert(CleanupNonStarportOptionParser.parse(args) === None)
assert(CleanupUnmanagedOptionParser.parse(args) === None)
}
}
"cutoffDate is not valid" should {
Expand All @@ -112,7 +112,7 @@ class CleanupNonStarportOptionParserSpec extends WordSpec {
)

"return None" in {
assert(CleanupNonStarportOptionParser.parse(args) === None)
assert(CleanupUnmanagedOptionParser.parse(args) === None)
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion version.sbt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
version in ThisBuild := "5.10.0"
version in ThisBuild := "5.11.0"

0 comments on commit 37f6349

Please sign in to comment.