Skip to content

Commit

Permalink
fix(v2): Temporarily disabling list archival (#131)
Browse files Browse the repository at this point in the history
  • Loading branch information
T145 committed Jul 10, 2024
1 parent ce4cfb8 commit 445acd2
Showing 1 changed file with 12 additions and 12 deletions.
24 changes: 12 additions & 12 deletions scripts/v2/build_lists.bash
Original file line number Diff line number Diff line change
Expand Up @@ -83,18 +83,18 @@ main() {
echo "[INFO] Processing method: ${method}"

set +e # Temporarily disable strict fail, in case web requests fail
echo "[INFO] Archiving ${method} lists..."

: >"$ARCHIVE"

# This will archive inactive lists too
jaq -r --arg method "$method" 'to_entries[] |
select(.value.method == $method).value.mirrors[] as $mirror |
("\(.key)#\($mirror)")' data/v2/manifest.json |
while IFS='#' read -r key url; do
curl -sSLI "https://web.archive.org/save/${url}" |
awk -v key="$key" '$1~/^location:$/{print key,$2}' >>"$ARCHIVE"
done
# echo "[INFO] Archiving ${method} lists..."

# : >"$ARCHIVE"

# # This will archive inactive lists too
# jaq -r --arg method "$method" 'to_entries[] |
# select(.value.method == $method).value.mirrors[] as $mirror |
# ("\(.key)#\($mirror)")' data/v2/manifest.json |
# while IFS='#' read -r key url; do
# curl -sSLI "https://web.archive.org/save/${url}" |
# awk -v key="$key" '$1~/^location:$/{print key,$2}' >>"$ARCHIVE"
# done

echo "[INFO] Downloading ${method} lists..."

Expand Down

0 comments on commit 445acd2

Please sign in to comment.