phish-download-show

#!/usr/bin/env nix-shell
#!/usr/bin/env bash
#! nix-shell -i bash -p bash coreutils curl diffutils jq libarchive pandoc tageditor python3Packages.ftfy
# shellcheck shell=bash

# keep-sorted start skip_lines=1 prefix_order=type,,>,||
type \
    bsdtar \
    cmp \
    curl \
    jq \
    mktemp \
    pandoc \
    tageditor \
    >/dev/null \
    || exit 1
# keep-sorted end

set -euo pipefail

usage() {
    # shellcheck disable=SC2059
    [[ "$#" -eq 0 ]] || printf "$@" >&2

    cat >&2 <<EOF
usage: ${0##*/} [-Tkvz] <show>

Download a show from Phish.in to the working directory, improving the
downloaded ZIP archive with better tagging, and extracting it to an
appropriately named subdirectory of the working directory. After
downloading, improving tags, and extracting the archive, print the
extracted directory path on standard output.

options:
    -T              Don't do any automatic tag editing.
    -k              Keep downloaded archive, don't delete it after
                    extracting it to the working directory.
    -v              Be more verbose, printing commands as they are ran.
    -z              Do not extract the archive; download it and exit.
    <show>          An ISO formatted date (ex. 1983-12-02).

see also: the Phish.net project <https://phish.net>,
          the Phish.in project <https://phish.in>.

Kylie McClain <kylie@somas.is>
EOF
    [[ "$#" -eq 0 ]] || exit 1
    exit 64 # EX_USAGE
}

edo() {
    local arg string
    string="$"
    for arg; do
        if [[ "${arg@Q}" == "'$arg'" ]] && ! [[ "${arg}" =~ [[:blank:]] ]]; then
            string+=" $arg"
        else
            string+=" ${arg@Q}"
        fi
    done

    if [[ "${verbosity}" -ge 1 ]]; then
        printf '%s\n' "$string" >&2 || :
    fi

    "$@"
}

curl() {
    command curl --location --user-agent 'phish-cli <kylie@somas.is>' "$@"
}

phishin() {
    jq -cr "$@" <<<"${phishin_show_data}"
}

edit_tags=true
keep_archive=false
only_download=false
verbosity=0
while getopts :Tkvz opt; do
    case "$opt" in
        T) edit_tags=false ;;
        k) keep_archive=true ;;
        v) verbosity=$((verbosity + 1)) ;;
        z) only_download=true ;;
        ?) usage '' ;;
    esac
done
shift $((OPTIND - 1))

[[ "${1:-}" == --help ]] && usage

[[ "$#" -eq 1 ]] || usage 'error: incorrect number of arguments (1 required)\n'

[[ "$1" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] \
    || usage 'error: invalid show argument format (YYYY-MM-DD required)\n'

show_date="$1"
shift

# Request that Phish.in generate a ZIP archive of the album, polling until it
# is actually made available to download if we end up making it generate one.
_said=false
while :; do
    request_album_zip=$(
        jq -nc '{date:$show_date}' --arg show_date "$show_date" \
            | curl --no-progress-meter --json @- \
                'https://phish.in/api/v2/shows/request_album_zip'
    )

    if jq -re '.message == "Album already generated"' <<<"${request_album_zip}" >/dev/null; then
        break
    else
        if [[ "$_said" == false ]]; then
            printf 'waiting for Phish.in to finish generating the album...\n' >&2
            _said=true
        fi

        sleep 15
    fi
done

# shellcheck disable=SC2310
if ! phishin_show_data=$(curl --no-progress-meter --fail "https://phish.in/api/v2/shows/$show_date"); then
    printf 'error: failed to download show data for %s\n' "${show_date}" >&2
    exit 1
fi

near_show_data=

# Collapse the data a bit and create a nice filename
# ex. Phish - 2000-05-23 Roseland Ballroom, New York, NY [SBD] [Complete]
show_download=$(
    phishin '{
        album_zip_url,
        updated_at,
        filename: (
            [
                "Phish - \(.date) \(.venue_name), \(.venue.location)",
                ((
                    ( .tags | map(.name))
                        + [
                            (.audio_status
                                | (
                                    (.[0:1] | ascii_upcase)
                                        + .[1:]
                                )
                            )
                        ]
                ) | map(" [\(.)]") | join(""))
            ] | join("")
        )
    }'
)

filename=$(jq -r .filename <<<"${show_download}")
directory="${PWD}/${filename}"
zip="${directory}.zip"

album_zip_url=$(jq -r .album_zip_url <<<"${show_download}")

near_show_data=$(
    if [[ -f "${directory}"/phishin.json ]]; then
        cat "${directory}"/phishin.json
    elif [[ -f "${zip}" ]]; then
        # Get downloaded ZIP's phishin.json and output to stdout
        edo bsdtar -Oxf "${zip}" '*/phishin.json'
    fi
)

# If the far and near show data differs (either $near_show_data is empty and
# far is not or whatever) then we need to download the show again.
# shellcheck disable=SC2312
if
    [[ -n "${near_show_data}" ]] \
        && [[ -d "${directory}" ]] \
        && ! cmp -s <(jq -c <<<"${phishin_show_data}" || printf '{}') <(jq -c <<<"${near_show_data}" || printf '{}')
then
    # and if none of this is true, then we don't need to do anything!
    printf '%s: downloaded version is up to date\n' "${show_date}" >&2
    printf '%s\n' "${directory}"
    exit 0
fi

if ! [[ -d "${directory}" ]]; then
    # Download the album, continuing an unfinished previous download if it exists.
    zip_raw="${zip%.zip}.raw.zip"
    if ! [[ -s "${zip_raw}" ]] && ! [[ -s "${zip}" ]]; then
        # Fetch the ZIP from Phish.in, with progress, and continuing if
        # an unfinished download exists.

        edo curl \
            --location \
            --fail \
            -# \
            -o "${zip_raw}".part \
            --continue-at - \
            "${album_zip_url}" || exit 1

        edo mv "${zip_raw}.part" "${zip_raw}"
    fi

    [[ "$only_download" == false ]] || exit

    # Take time to re-create the ZIP archive generated by Phish.in, and slip in a
    # phishin.json data file that contains the show data from the API while we're
    # at it.
    if [[ -s "$zip_raw" ]] && ! [[ -s "${zip}" ]]; then
        repack=$(mktemp -d)
        repack_album="${repack}/${filename}"
        edo mkdir -p "$repack_album"

        # This is done before extracting in hopes that maybe some day Phish.in
        # will slip a phishin.json into the show archive itself.
        printf '%s' "${phishin_show_data:?}" >"$repack_album"/phishin.json

        # Extract the unedited archive into the album directory,
        # and then remove the archive from the working directory.
        edo bsdtar -C "$repack_album" -xf "${zip_raw}"

        # Use a more default cover filename, and don't include the captionless
        # version of the AI slop art.
        edo rm -f "$repack_album"/cover_art.jpg
        edo mv "$repack_album"/album_cover.jpg "$repack_album"/cover.jpg

        # Repack it, with a leading subdirectory
        edo bsdtar -C "$repack" -cf "${zip}".part "${filename}"/
        edo mv "${zip}".part "${zip}"

        edo rm -rf "$repack"
    fi

    [[ -e "$zip" ]] || exit 1

    edo rm -f "$zip_raw"

    if ! edo bsdtar -xf "${zip}"; then
        printf 'error: failed while extracting downloaded album (%s)\n' "${zip@Q}" >&2
        exit 1
    fi
fi

[[ "$keep_archive" == true ]] || edo rm -f "${zip}"

# Process the show data to get data pertaining to the track file in question,
# and output it all in format suitable for `eval`.
# shellcheck disable=SC2016
jq_script='
    .tracks[$file_i]
        | {
            phishin_track: tojson,
            album: "\(.show_date): \(.venue_name), \(.venue_location)",
            show_date,
            title,
            set_name,
            track_url: "https://phish.in/\(.show_date)/\(.slug)",
            comments: [
                (.tags | map(select(.notes != null) | "\(.name): \(.notes)") | join("\n")),
                $phishNetSetlistNotes
            ] | map(select(. != ""))
            | join("\n")
        }
        | to_entries
        | map("\(.key)=\(.value|@sh)")[]

'

# Work through the downloaded MP3s in the archive, and modify the tags
# given to be nicer and more informative by default; set disc numbers to be
# the show set number; set the recording date to be correct; set a disc subtitle
# to the set name; set the publisher webpage to the track's Phish.in permalink;
# put Phish.in track tags (Jamcharts, Teases, etc.) in the comments.
set_name=
file_i=0
disc=1

if [[ "$edit_tags" == true ]]; then
    # See "ftfy" comment in `phish-show-notes` for why this is necessary.
    # Pandoc is used where something smaller would suffice (like recode)
    # for replacing the escaped HTML-entities because pandoc is also used
    # by `phish-show-notes`.
    phishnet_setlist_notes=$(
        phish-show-notes -j "${show_date}" \
            | jq -r '.setlist_notes // ""' \
            | ftfy -g \
            | pandoc -f html -t plain --wrap=none
    )

    for file in "${directory}"/*.mp3; do
        if ! [[ -e "$file" ]]; then
            printf 'error: file %s does not exist\n' "${file@Q}" >&2
            exit 1
        fi

        jq_eval=$(jq -r \
            --arg phishNetSetlistNotes "${phishnet_setlist_notes}" \
            --argjson file_i "$file_i" \
            "$jq_script" <<<"${phishin_show_data}")
        eval "$jq_eval"

        : "${album:?}"
        : "${title:?}"
        : "${show_date:?}"
        : "${set_name:?}"
        : "${track_url:?}"
        : "${comments?}"

        last_set_name="${last_set_name:-${set_name:-}}"

        if [[ "${set_name}" != "${last_set_name}" ]]; then
            disc=$((disc + 1))
        fi

        {
            # What do we actually do to the tags compared to Phish.in?
            #
            # Phish.in includes ID3v1 tags, which have enough drawbacks that I
            # don't want anything trying to use them (such as the tag value's
            # fifteen character limit). So we remove those, and also upgrade
            # the ID3v2.3 tags to 2.4 while we're at it.
            #
            # Given the file "Phish - 2012-08-24 Oak Mountain Amphitheatre,
            # Pelham, AL [Complete]/14 Rock and Roll.mp3", the tag changes are:
            # album="2012-08-24 Oak Mountain Amphitheatre"
            #   -> album="2012-08-24: Oak Mountain Amphitheatre, Pelham, AL"
            # comment="https://phish.in for more"
            #   -> comment="Jamcharts: Very good, multi-part jam. Begins in a
            #      snappy, rocking mode with a good \"Live And Let Die\" tease,
            #      grows in power, then mellows with a -> to \"Lizards.\"\n
            #      Tease: Live and Let Die by Paul McCartney and Wings\n
            #      Tease: Twenty Years Later"
            # recorddate=2012
            #   -> recorddate=2012-08-24
            # !publisherwebpage
            #   -> publisherwebpage="https://phish.in/2012-08-24/rock-and-roll"
            # !disc                  [Phish.in's MP3s in fact have no information
            #   -> disc=2             pertaining to which set they are part of]
            # !discname (id3:TSST)   [tageditor doesn't seem to know about TSST,
            #   -> discname="Set 2"   the disc subtitle tag in ID3.]
            tags=(
                album="${album}"
                title="${title}"
                track=$((file_i + 1))
                comment="${comments}"
                recorddate="${show_date}"
                publisherwebpage="${track_url}"
                disk="${disc}"
                id3:TSST="${set_name}"
            )

            if [[ -e "${file}" ]] && ! [[ -s "${file}" ]]; then
                # Phish.in gives us empty MP3 files in the downloaded archive,
                # for tracks that should exist in a set, but do not exist as
                # part of the recording they have. Consequentially, beets then
                # will report an "unreadable file"; so, we should warn the user,
                # and then delete the empty file.
                printf 'warning: skipping track %i (%s); show recording only has partial audio\n' "$((file_i + 1))" "${set_name}: ${title}" >&2
                edo rm -f "${file}"
            else
                edo tageditor set "${tags[@]}" \
                    --id3v1-usage never \
                    --id3v2-version 4 \
                    --quiet \
                    --verbose \
                    --files "${file}" \
                    && rm -f "${file}".bak
            fi
        } >/dev/null

        last_set_name="${set_name}"
        file_i=$((file_i + 1))
    done
fi

[[ "$keep_archive" == true ]] || edo rm -f "${zip}"

# Everything went fine so now print the downloaded show on stdout to indicate
# to any other tools listening that the show in question has been downloaded.
printf '%s\n' "${directory}"