darkvisitors_cron.sh

#! /usr/bin/env bash

[[ -z "${AGENT_TYPES}" ]] && AGENT_TYPES="[\"AI Assistant\", \"AI Data Scraper\", \"AI Search Crawler\", \"Undocumented AI Agent\"]"

usage(){
        echo -e "Usage:\n
    -t | DARKVISITORS_TOKEN= Site's access token
    -d | DARKVISITORS_DISALLOW= A string specifying which URLs are disallowed. Defaults to / to disallow all URLs.
    -a | AGENT_TYPES= An array of agent types. Agent types include AI Assistant, AI Data Scraper, AI Search Crawler, and Undocumented AI Agent. Defaults to ${AGENT_TYPES}
    -c | CURL_ARGS= Args to pass to curl

example usage: ${0} -t acab420c-dca0-49e4-8e45-db647808fcde -a \'[\"AI Assistant\", \"AI Data Scraper\", \"AI Search Crawler\", \"Undocumented AI Agent\"]\' -d / -c -v

darkvisitors docs: https://darkvisitors.com/docs/robots-txt"
        exit 1
}
if [[ "${#}" == "0" ]]
then
        usage
fi

POSITIONAL=()
while [[ $# -gt 0 ]]; do
        key="$1"
        case $key in
        -t)
                DARKVISITORS_TOKEN="${2}"
                shift # past argument
                shift # past value
        ;;
        -a)
                AGENT_TYPES="${2}"
                shift # past argument
                shift # past value
        ;;
        -d)
                DARKVISITORS_DISALLOW="${2}"
                shift # past argument
                shift # past value
        ;;
        -c)
                CURL_ARGS="${CURL_ARGS} ${2}"
                shift # past argument
                shift # past value
        ;;
        esac
done
set -- "${POSITIONAL[@]}" # restore positional parameters

if [[ -z "${DARKVISITORS_TOKEN}" ]] || [[ -z "${AGENT_TYPES}" ]]
then
        usage
fi

if [[ -z ${DARKVISITORS_DISALLOW} ]]
then
        DARKVISITORS_DISALLOW="/"
fi

gen_json(){
cat <<EOF
{
"agent_types": ${AGENT_TYPES},
"disallow": "${DARKVISITORS_DISALLOW}"
}
EOF
}

curl -f ${CURL_ARGS} -o robots.txt.new -X POST -H 'Content-Type: application/json' -H "Authorization: Bearer ${DARKVISITORS_TOKEN}" https://api.darkvisitors.com/robots-txts --data "$(gen_json)"
if [[ "${?}" == 0 ]]
then
        mv -fv robots.txt.new robots.txt
else
        [[ -f robots.txt.new ]] && rm -v robots.txt.new
fi