Merge pull request #623 from HazyResearch/fix-gpfdist-to-pick-availab…

…le-port

Fixes gpfdist to pick available port with randomized base port

database/db-driver/greenplum/db-unload

@@ -6,15 +6,16 @@
 # parallel connections, which MAY RESULT IN INCORRECT RESULT UNLESS QUERY IS
 # TOTALLY ORDERED BY COLUMNS WITH NO DUPLICATE VALUES!
 #
-# GPFDIST_PORT_BASE (defaults to 12000) is the base port number for the gpfdist
-# processes.
+# GPFDIST_PORT_BASE is the base port number for the gpfdist processes.
+# (defaults to a random port in range 10000-59900)
 # GPFDIST_MAX_LENGTH (defaults to 1MiB) may need to be increased for larger
 # records to be unloaded through gpfdist.
 # GPFDIST_LOG_PATH can be set to a path prefix for logging.
 ##
 set -eu
 
-: ${GPFDIST_DISABLE:=} ${GPFDIST_PORT_BASE:=12000} ${GPFDIST_MAX_LENGTH:=$((2**20))} ${GPFDIST_LOG_PATH:=}
+: ${GPFDIST_DISABLE:=} ${GPFDIST_MAX_LENGTH:=$((2**20))} ${GPFDIST_LOG_PATH:=}
+: ${GPFDIST_PORT_BASE:=$(( 100*($RANDOM % 500 + 100) ))} # randomized default base ports
 
 [[ $# -gt 0 ]] || usage "$0" "Missing QUERY"
 query=$1; shift
@@ -70,12 +71,14 @@ case $format in
                 sinkDir=$(dirname "$sink") sinkName=$(basename "$sink")
                 mkdir -p "$sinkDir"
                 spawn_gpfdist() {
+                    # skip over ports already taken
+                    while nc -z "$sinkHost" "$port"; do let ++port; done
                     gpfdist -p $port \
                         ${GPFDIST_MAX_LENGTH:+-m $GPFDIST_MAX_LENGTH} \
                         ${GPFDIST_LOG_PATH:+-l ${GPFDIST_LOG_PATH}.$i.log} &>/dev/null \
                         "$@" &
                     echo $! >>"$tmpDir"/gpfdist-$i.pid
-                    # TODO cope with gpfdist failure, e.g., port number collision
+                    # TODO cope with gpfdist failure, e.g., when another process suddenly tried to listen to the picked port number
                 }
                 if ${DEEPDIVE_SHOW_PROGRESS:-true}; then
                     spawn_gpfdist -d "$tmpDir"