Mirror of A black hole for Internet advertisements
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

gravity.sh 23KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694
  1. #!/usr/bin/env bash
  2. # shellcheck disable=SC1090
  3. # Pi-hole: A black hole for Internet advertisements
  4. # (c) 2017 Pi-hole, LLC (https://pi-hole.net)
  5. # Network-wide ad blocking via your own hardware.
  6. #
  7. # Usage: "pihole -g"
  8. # Compiles a list of ad-serving domains by downloading them from multiple sources
  9. #
  10. # This file is copyright under the latest version of the EUPL.
  11. # Please see LICENSE file for your rights under this license.
  12. export LC_ALL=C
  13. coltable="/opt/pihole/COL_TABLE"
  14. source "${coltable}"
  15. regexconverter="/opt/pihole/wildcard_regex_converter.sh"
  16. source "${regexconverter}"
  17. basename="pihole"
  18. PIHOLE_COMMAND="/usr/local/bin/${basename}"
  19. piholeDir="/etc/${basename}"
  20. adListFile="${piholeDir}/adlists.list"
  21. adListDefault="${piholeDir}/adlists.default"
  22. whitelistFile="${piholeDir}/whitelist.txt"
  23. blacklistFile="${piholeDir}/blacklist.txt"
  24. regexFile="${piholeDir}/regex.list"
  25. adList="${piholeDir}/gravity.list"
  26. blackList="${piholeDir}/black.list"
  27. localList="${piholeDir}/local.list"
  28. VPNList="/etc/openvpn/ipp.txt"
  29. domainsExtension="domains"
  30. matterAndLight="${basename}.0.matterandlight.txt"
  31. parsedMatter="${basename}.1.parsedmatter.txt"
  32. whitelistMatter="${basename}.2.whitelistmatter.txt"
  33. accretionDisc="${basename}.3.accretionDisc.txt"
  34. preEventHorizon="list.preEventHorizon"
  35. skipDownload="false"
  36. resolver="pihole-FTL"
  37. haveSourceUrls=true
  38. # Source setupVars from install script
  39. setupVars="${piholeDir}/setupVars.conf"
  40. if [[ -f "${setupVars}" ]];then
  41. source "${setupVars}"
  42. # Remove CIDR mask from IPv4/6 addresses
  43. IPV4_ADDRESS="${IPV4_ADDRESS%/*}"
  44. IPV6_ADDRESS="${IPV6_ADDRESS%/*}"
  45. # Determine if IPv4/6 addresses exist
  46. if [[ -z "${IPV4_ADDRESS}" ]] && [[ -z "${IPV6_ADDRESS}" ]]; then
  47. echo -e " ${COL_LIGHT_RED}No IP addresses found! Please run 'pihole -r' to reconfigure${COL_NC}"
  48. exit 1
  49. fi
  50. else
  51. echo -e " ${COL_LIGHT_RED}Installation Failure: ${setupVars} does not exist! ${COL_NC}
  52. Please run 'pihole -r', and choose the 'reconfigure' option to fix."
  53. exit 1
  54. fi
  55. # Source pihole-FTL from install script
  56. pihole_FTL="${piholeDir}/pihole-FTL.conf"
  57. if [[ -f "${pihole_FTL}" ]]; then
  58. source "${pihole_FTL}"
  59. fi
  60. if [[ -z "${BLOCKINGMODE}" ]] ; then
  61. BLOCKINGMODE="NULL"
  62. fi
  63. # Determine if superseded pihole.conf exists
  64. if [[ -r "${piholeDir}/pihole.conf" ]]; then
  65. echo -e " ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}"
  66. fi
  67. # Determine if Pi-hole blocking is disabled
  68. # If this is the case, we want to update
  69. # gravity.list.bck and black.list.bck instead of
  70. # gravity.list and black.list
  71. detect_pihole_blocking_status() {
  72. if [[ "${BLOCKING_ENABLED}" == false ]]; then
  73. echo -e " ${INFO} Pi-hole blocking is disabled"
  74. adList="${adList}.bck"
  75. blackList="${blackList}.bck"
  76. else
  77. echo -e " ${INFO} Pi-hole blocking is enabled"
  78. fi
  79. }
  80. # Determine if DNS resolution is available before proceeding
  81. gravity_CheckDNSResolutionAvailable() {
  82. local lookupDomain="pi.hole"
  83. # Determine if $localList does not exist
  84. if [[ ! -e "${localList}" ]]; then
  85. lookupDomain="raw.githubusercontent.com"
  86. fi
  87. # Determine if $lookupDomain is resolvable
  88. if timeout 1 getent hosts "${lookupDomain}" &> /dev/null; then
  89. # Print confirmation of resolvability if it had previously failed
  90. if [[ -n "${secs:-}" ]]; then
  91. echo -e "${OVER} ${TICK} DNS resolution is now available\\n"
  92. fi
  93. return 0
  94. elif [[ -n "${secs:-}" ]]; then
  95. echo -e "${OVER} ${CROSS} DNS resolution is not available"
  96. exit 1
  97. fi
  98. # If the /etc/resolv.conf contains resolvers other than 127.0.0.1 then the local dnsmasq will not be queried and pi.hole is NXDOMAIN.
  99. # This means that even though name resolution is working, the getent hosts check fails and the holddown timer keeps ticking and eventualy fails
  100. # So we check the output of the last command and if it failed, attempt to use dig +short as a fallback
  101. if timeout 1 dig +short "${lookupDomain}" &> /dev/null; then
  102. if [[ -n "${secs:-}" ]]; then
  103. echo -e "${OVER} ${TICK} DNS resolution is now available\\n"
  104. fi
  105. return 0
  106. elif [[ -n "${secs:-}" ]]; then
  107. echo -e "${OVER} ${CROSS} DNS resolution is not available"
  108. exit 1
  109. fi
  110. # Determine error output message
  111. if pidof ${resolver} &> /dev/null; then
  112. echo -e " ${CROSS} DNS resolution is currently unavailable"
  113. else
  114. echo -e " ${CROSS} DNS service is not running"
  115. "${PIHOLE_COMMAND}" restartdns
  116. fi
  117. # Ensure DNS server is given time to be resolvable
  118. secs="120"
  119. echo -ne " ${INFO} Time until retry: ${secs}"
  120. until timeout 1 getent hosts "${lookupDomain}" &> /dev/null; do
  121. [[ "${secs:-}" -eq 0 ]] && break
  122. echo -ne "${OVER} ${INFO} Time until retry: ${secs}"
  123. : $((secs--))
  124. sleep 1
  125. done
  126. # Try again
  127. gravity_CheckDNSResolutionAvailable
  128. }
  129. # Retrieve blocklist URLs and parse domains from adlists.list
  130. gravity_GetBlocklistUrls() {
  131. echo -e " ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..."
  132. if [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then
  133. # Remove superceded $adListDefault file
  134. rm "${adListDefault}" 2> /dev/null || \
  135. echo -e " ${CROSS} Unable to remove ${adListDefault}"
  136. fi
  137. # Retrieve source URLs from $adListFile
  138. # Logic: Remove comments and empty lines
  139. mapfile -t sources <<< "$(grep -v -E "^(#|$)" "${adListFile}" 2> /dev/null)"
  140. # Parse source domains from $sources
  141. mapfile -t sourceDomains <<< "$(
  142. # Logic: Split by folder/port
  143. awk -F '[/:]' '{
  144. # Remove URL protocol & optional username:password@
  145. gsub(/(.*:\/\/|.*:.*@)/, "", $0)
  146. if(length($1)>0){print $1}
  147. else {print "local"}
  148. }' <<< "$(printf '%s\n' "${sources[@]}")" 2> /dev/null
  149. )"
  150. local str="Pulling blocklist source list into range"
  151. if [[ -n "${sources[*]}" ]] && [[ -n "${sourceDomains[*]}" ]]; then
  152. echo -e "${OVER} ${TICK} ${str}"
  153. else
  154. echo -e "${OVER} ${CROSS} ${str}"
  155. echo -e " ${INFO} No source list found, or it is empty"
  156. echo ""
  157. haveSourceUrls=false
  158. fi
  159. }
  160. # Define options for when retrieving blocklists
  161. gravity_SetDownloadOptions() {
  162. local url domain agent cmd_ext str
  163. echo ""
  164. # Loop through $sources and download each one
  165. for ((i = 0; i < "${#sources[@]}"; i++)); do
  166. url="${sources[$i]}"
  167. domain="${sourceDomains[$i]}"
  168. # Save the file as list.#.domain
  169. saveLocation="${piholeDir}/list.${i}.${domain}.${domainsExtension}"
  170. activeDomains[$i]="${saveLocation}"
  171. # Default user-agent (for Cloudflare's Browser Integrity Check: https://support.cloudflare.com/hc/en-us/articles/200170086-What-does-the-Browser-Integrity-Check-do-)
  172. agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36"
  173. # Provide special commands for blocklists which may need them
  174. case "${domain}" in
  175. "pgl.yoyo.org") cmd_ext="-d mimetype=plaintext -d hostformat=hosts";;
  176. *) cmd_ext="";;
  177. esac
  178. if [[ "${skipDownload}" == false ]]; then
  179. echo -e " ${INFO} Target: ${domain} (${url##*/})"
  180. gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}"
  181. echo ""
  182. fi
  183. done
  184. gravity_Blackbody=true
  185. }
  186. # Download specified URL and perform checks on HTTP status and file content
  187. gravity_DownloadBlocklistFromUrl() {
  188. local url="${1}" cmd_ext="${2}" agent="${3}" heisenbergCompensator="" patternBuffer str httpCode success=""
  189. # Create temp file to store content on disk instead of RAM
  190. patternBuffer=$(mktemp -p "/tmp" --suffix=".phgpb")
  191. # Determine if $saveLocation has read permission
  192. if [[ -r "${saveLocation}" && $url != "file"* ]]; then
  193. # Have curl determine if a remote file has been modified since last retrieval
  194. # Uses "Last-Modified" header, which certain web servers do not provide (e.g: raw github urls)
  195. # Note: Don't do this for local files, always download them
  196. heisenbergCompensator="-z ${saveLocation}"
  197. fi
  198. str="Status:"
  199. echo -ne " ${INFO} ${str} Pending..."
  200. blocked=false
  201. case $BLOCKINGMODE in
  202. "IP-NODATA-AAAA"|"IP")
  203. if [[ $(dig "${domain}" +short | grep "${IPV4_ADDRESS}" -c) -ge 1 ]]; then
  204. blocked=true
  205. fi;;
  206. "NXDOMAIN")
  207. if [[ $(dig "${domain}" | grep "NXDOMAIN" -c) -ge 1 ]]; then
  208. blocked=true
  209. fi;;
  210. "NULL"|*)
  211. if [[ $(dig "${domain}" +short | grep "0.0.0.0" -c) -ge 1 ]]; then
  212. blocked=true
  213. fi;;
  214. esac
  215. if [[ "${blocked}" == true ]]; then
  216. printf -v ip_addr "%s" "${PIHOLE_DNS_1%#*}"
  217. if [[ ${PIHOLE_DNS_1} != *"#"* ]]; then
  218. port=53
  219. else
  220. printf -v port "%s" "${PIHOLE_DNS_1#*#}"
  221. fi
  222. ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}" | tail -1)
  223. if [[ $(echo "${url}" | awk -F '://' '{print $1}') = "https" ]]; then
  224. port=443;
  225. else port=80
  226. fi
  227. bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}')
  228. echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${PIHOLE_DNS_1} to download ${url}";
  229. echo -ne " ${INFO} ${str} Pending..."
  230. cmd_ext="--resolve $domain:$port:$ip $cmd_ext"
  231. fi
  232. # shellcheck disable=SC2086
  233. httpCode=$(curl -s -L ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${patternBuffer}" 2> /dev/null)
  234. case $url in
  235. # Did we "download" a local file?
  236. "file"*)
  237. if [[ -s "${patternBuffer}" ]]; then
  238. echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true
  239. else
  240. echo -e "${OVER} ${CROSS} ${str} Not found / empty list"
  241. fi;;
  242. # Did we "download" a remote file?
  243. *)
  244. # Determine "Status:" output based on HTTP response
  245. case "${httpCode}" in
  246. "200") echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true;;
  247. "304") echo -e "${OVER} ${TICK} ${str} No changes detected"; success=true;;
  248. "000") echo -e "${OVER} ${CROSS} ${str} Connection Refused";;
  249. "403") echo -e "${OVER} ${CROSS} ${str} Forbidden";;
  250. "404") echo -e "${OVER} ${CROSS} ${str} Not found";;
  251. "408") echo -e "${OVER} ${CROSS} ${str} Time-out";;
  252. "451") echo -e "${OVER} ${CROSS} ${str} Unavailable For Legal Reasons";;
  253. "500") echo -e "${OVER} ${CROSS} ${str} Internal Server Error";;
  254. "504") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Gateway)";;
  255. "521") echo -e "${OVER} ${CROSS} ${str} Web Server Is Down (Cloudflare)";;
  256. "522") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Cloudflare)";;
  257. * ) echo -e "${OVER} ${CROSS} ${str} ${url} (${httpCode})";;
  258. esac;;
  259. esac
  260. # Determine if the blocklist was downloaded and saved correctly
  261. if [[ "${success}" == true ]]; then
  262. if [[ "${httpCode}" == "304" ]]; then
  263. : # Do not attempt to re-parse file
  264. # Check if $patternbuffer is a non-zero length file
  265. elif [[ -s "${patternBuffer}" ]]; then
  266. # Determine if blocklist is non-standard and parse as appropriate
  267. gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}"
  268. else
  269. # Fall back to previously cached list if $patternBuffer is empty
  270. echo -e " ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
  271. fi
  272. else
  273. # Determine if cached list has read permission
  274. if [[ -r "${saveLocation}" ]]; then
  275. echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
  276. else
  277. echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}"
  278. fi
  279. fi
  280. }
  281. # Parse source files into domains format
  282. gravity_ParseFileIntoDomains() {
  283. local source="${1}" destination="${2}" firstLine abpFilter
  284. # Determine if we are parsing a consolidated list
  285. if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then
  286. # Remove comments and print only the domain name
  287. # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious
  288. # This helps with that and makes it easier to read
  289. # It also helps with debugging so each stage of the script can be researched more in depth
  290. # Awk -F splits on given IFS, we grab the right hand side (chops trailing #coments and /'s to grab the domain only.
  291. # Last awk command takes non-commented lines and if they have 2 fields, take the right field (the domain) and leave
  292. # the left (IP address), otherwise grab the single field.
  293. < ${source} awk -F '#' '{print $1}' | \
  294. awk -F '/' '{print $1}' | \
  295. awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' | \
  296. sed -nr -e 's/\.{2,}/./g' -e '/\./p' > ${destination}
  297. return 0
  298. fi
  299. # Individual file parsing: Keep comments, while parsing domains from each line
  300. # We keep comments to respect the list maintainer's licensing
  301. read -r firstLine < "${source}"
  302. # Determine how to parse individual source file formats
  303. if [[ "${firstLine,,}" =~ (adblock|ublock|^!) ]]; then
  304. # Compare $firstLine against lower case words found in Adblock lists
  305. echo -e " ${CROSS} Format: Adblock (list type not supported)"
  306. elif grep -q "^address=/" "${source}" &> /dev/null; then
  307. # Parse Dnsmasq format lists
  308. echo -e " ${CROSS} Format: Dnsmasq (list type not supported)"
  309. elif grep -q -E "^https?://" "${source}" &> /dev/null; then
  310. # Parse URL list if source file contains "http://" or "https://"
  311. # Scanning for "^IPv4$" is too slow with large (1M) lists on low-end hardware
  312. echo -ne " ${INFO} Format: URL"
  313. awk '
  314. # Remove URL scheme, optional "username:password@", and ":?/;"
  315. # The scheme must be matched carefully to avoid blocking the wrong URL
  316. # in cases like:
  317. # http://www.evil.com?http://www.good.com
  318. # See RFC 3986 section 3.1 for details.
  319. /[:?\/;]/ { gsub(/(^[a-zA-Z][a-zA-Z0-9+.-]*:\/\/(.*:.*@)?|[:?\/;].*)/, "", $0) }
  320. # Skip lines which are only IPv4 addresses
  321. /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/ { next }
  322. # Print if nonempty
  323. length { print }
  324. ' "${source}" 2> /dev/null > "${destination}"
  325. echo -e "${OVER} ${TICK} Format: URL"
  326. else
  327. # Default: Keep hosts/domains file in same format as it was downloaded
  328. output=$( { mv "${source}" "${destination}"; } 2>&1 )
  329. if [[ ! -e "${destination}" ]]; then
  330. echo -e "\\n ${CROSS} Unable to move tmp file to ${piholeDir}
  331. ${output}"
  332. gravity_Cleanup "error"
  333. fi
  334. fi
  335. }
  336. # Create (unfiltered) "Matter and Light" consolidated list
  337. gravity_ConsolidateDownloadedBlocklists() {
  338. local str lastLine
  339. str="Consolidating blocklists"
  340. if [[ "${haveSourceUrls}" == true ]]; then
  341. echo -ne " ${INFO} ${str}..."
  342. fi
  343. # Empty $matterAndLight if it already exists, otherwise, create it
  344. : > "${piholeDir}/${matterAndLight}"
  345. # Loop through each *.domains file
  346. for i in "${activeDomains[@]}"; do
  347. # Determine if file has read permissions, as download might have failed
  348. if [[ -r "${i}" ]]; then
  349. # Remove windows CRs from file, convert list to lower case, and append into $matterAndLight
  350. tr -d '\r' < "${i}" | tr '[:upper:]' '[:lower:]' >> "${piholeDir}/${matterAndLight}"
  351. # Ensure that the first line of a new list is on a new line
  352. lastLine=$(tail -1 "${piholeDir}/${matterAndLight}")
  353. if [[ "${#lastLine}" -gt 0 ]]; then
  354. echo "" >> "${piholeDir}/${matterAndLight}"
  355. fi
  356. fi
  357. done
  358. if [[ "${haveSourceUrls}" == true ]]; then
  359. echo -e "${OVER} ${TICK} ${str}"
  360. fi
  361. }
  362. # Parse consolidated list into (filtered, unique) domains-only format
  363. gravity_SortAndFilterConsolidatedList() {
  364. local str num
  365. str="Extracting domains from blocklists"
  366. if [[ "${haveSourceUrls}" == true ]]; then
  367. echo -ne " ${INFO} ${str}..."
  368. fi
  369. # Parse into hosts file
  370. gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}"
  371. # Format $parsedMatter line total as currency
  372. num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")")
  373. if [[ "${haveSourceUrls}" == true ]]; then
  374. echo -e "${OVER} ${TICK} ${str}"
  375. fi
  376. echo -e " ${INFO} Number of domains being pulled in by gravity: ${COL_BLUE}${num}${COL_NC}"
  377. str="Removing duplicate domains"
  378. if [[ "${haveSourceUrls}" == true ]]; then
  379. echo -ne " ${INFO} ${str}..."
  380. fi
  381. sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}"
  382. if [[ "${haveSourceUrls}" == true ]]; then
  383. echo -e "${OVER} ${TICK} ${str}"
  384. # Format $preEventHorizon line total as currency
  385. num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
  386. echo -e " ${INFO} Number of unique domains trapped in the Event Horizon: ${COL_BLUE}${num}${COL_NC}"
  387. fi
  388. }
  389. # Whitelist user-defined domains
  390. gravity_Whitelist() {
  391. local num str
  392. if [[ ! -f "${whitelistFile}" ]]; then
  393. echo -e " ${INFO} Nothing to whitelist!"
  394. return 0
  395. fi
  396. num=$(wc -l < "${whitelistFile}")
  397. str="Number of whitelisted domains: ${num}"
  398. echo -ne " ${INFO} ${str}..."
  399. # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in $whitelistFile
  400. comm -23 "${piholeDir}/${preEventHorizon}" <(sort "${whitelistFile}") > "${piholeDir}/${whitelistMatter}"
  401. echo -e "${OVER} ${INFO} ${str}"
  402. }
  403. # Output count of blacklisted domains and regex filters
  404. gravity_ShowBlockCount() {
  405. local num
  406. if [[ -f "${blacklistFile}" ]]; then
  407. num=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")")
  408. echo -e " ${INFO} Number of blacklisted domains: ${num}"
  409. fi
  410. if [[ -f "${regexFile}" ]]; then
  411. num=$(grep -cv "^#" "${regexFile}")
  412. echo -e " ${INFO} Number of regex filters: ${num}"
  413. fi
  414. }
  415. # Parse list of domains into hosts format
  416. gravity_ParseDomainsIntoHosts() {
  417. awk -v ipv4="$IPV4_ADDRESS" -v ipv6="$IPV6_ADDRESS" '{
  418. # Remove windows CR line endings
  419. sub(/\r$/, "")
  420. # Parse each line as "ipaddr domain"
  421. if(ipv6 && ipv4) {
  422. print ipv4" "$0"\n"ipv6" "$0
  423. } else if(!ipv6) {
  424. print ipv4" "$0
  425. } else {
  426. print ipv6" "$0
  427. }
  428. }' >> "${2}" < "${1}"
  429. }
  430. # Create "localhost" entries into hosts format
  431. gravity_ParseLocalDomains() {
  432. local hostname
  433. if [[ -s "/etc/hostname" ]]; then
  434. hostname=$(< "/etc/hostname")
  435. elif command -v hostname &> /dev/null; then
  436. hostname=$(hostname -f)
  437. else
  438. echo -e " ${CROSS} Unable to determine fully qualified domain name of host"
  439. return 0
  440. fi
  441. echo -e "${hostname}\\npi.hole" > "${localList}.tmp"
  442. # Empty $localList if it already exists, otherwise, create it
  443. : > "${localList}"
  444. gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}"
  445. # Add additional LAN hosts provided by OpenVPN (if available)
  446. if [[ -f "${VPNList}" ]]; then
  447. awk -F, '{printf $2"\t"$1".vpn\n"}' "${VPNList}" >> "${localList}"
  448. fi
  449. }
  450. # Create primary blacklist entries
  451. gravity_ParseBlacklistDomains() {
  452. local output status
  453. # Empty $accretionDisc if it already exists, otherwise, create it
  454. : > "${piholeDir}/${accretionDisc}"
  455. if [[ -f "${piholeDir}/${whitelistMatter}" ]]; then
  456. mv "${piholeDir}/${whitelistMatter}" "${piholeDir}/${accretionDisc}"
  457. else
  458. # There was no whitelist file, so use preEventHorizon instead of whitelistMatter.
  459. cp "${piholeDir}/${preEventHorizon}" "${piholeDir}/${accretionDisc}"
  460. fi
  461. # Move the file over as /etc/pihole/gravity.list so dnsmasq can use it
  462. output=$( { mv "${piholeDir}/${accretionDisc}" "${adList}"; } 2>&1 )
  463. status="$?"
  464. if [[ "${status}" -ne 0 ]]; then
  465. echo -e "\\n ${CROSS} Unable to move ${accretionDisc} from ${piholeDir}\\n ${output}"
  466. gravity_Cleanup "error"
  467. fi
  468. }
  469. # Create user-added blacklist entries
  470. gravity_ParseUserDomains() {
  471. if [[ ! -f "${blacklistFile}" ]]; then
  472. return 0
  473. fi
  474. # Copy the file over as /etc/pihole/black.list so dnsmasq can use it
  475. cp "${blacklistFile}" "${blackList}" 2> /dev/null || \
  476. echo -e "\\n ${CROSS} Unable to move ${blacklistFile##*/} to ${piholeDir}"
  477. }
  478. # Trap Ctrl-C
  479. gravity_Trap() {
  480. trap '{ echo -e "\\n\\n ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT
  481. }
  482. # Clean up after Gravity upon exit or cancellation
  483. gravity_Cleanup() {
  484. local error="${1:-}"
  485. str="Cleaning up stray matter"
  486. echo -ne " ${INFO} ${str}..."
  487. # Delete tmp content generated by Gravity
  488. rm ${piholeDir}/pihole.*.txt 2> /dev/null
  489. rm ${piholeDir}/*.tmp 2> /dev/null
  490. rm /tmp/*.phgpb 2> /dev/null
  491. # Ensure this function only runs when gravity_SetDownloadOptions() has completed
  492. if [[ "${gravity_Blackbody:-}" == true ]]; then
  493. # Remove any unused .domains files
  494. for file in ${piholeDir}/*.${domainsExtension}; do
  495. # If list is not in active array, then remove it
  496. if [[ ! "${activeDomains[*]}" == *"${file}"* ]]; then
  497. rm -f "${file}" 2> /dev/null || \
  498. echo -e " ${CROSS} Failed to remove ${file##*/}"
  499. fi
  500. done
  501. fi
  502. echo -e "${OVER} ${TICK} ${str}"
  503. # Only restart DNS service if offline
  504. if ! pidof ${resolver} &> /dev/null; then
  505. "${PIHOLE_COMMAND}" restartdns
  506. dnsWasOffline=true
  507. fi
  508. # Print Pi-hole status if an error occured
  509. if [[ -n "${error}" ]]; then
  510. "${PIHOLE_COMMAND}" status
  511. exit 1
  512. fi
  513. }
  514. helpFunc() {
  515. echo "Usage: pihole -g
  516. Update domains from blocklists specified in adlists.list
  517. Options:
  518. -f, --force Force the download of all specified blocklists
  519. -h, --help Show this help dialog"
  520. exit 0
  521. }
  522. for var in "$@"; do
  523. case "${var}" in
  524. "-f" | "--force" ) forceDelete=true;;
  525. "-h" | "--help" ) helpFunc;;
  526. "-sd" | "--skip-download" ) skipDownload=true;;
  527. "-b" | "--blacklist-only" ) listType="blacklist";;
  528. "-w" | "--whitelist-only" ) listType="whitelist";;
  529. "-wild" | "--wildcard-only" ) listType="wildcard"; dnsRestartType="restart";;
  530. esac
  531. done
  532. # Trap Ctrl-C
  533. gravity_Trap
  534. if [[ "${forceDelete:-}" == true ]]; then
  535. str="Deleting existing list cache"
  536. echo -ne "${INFO} ${str}..."
  537. rm /etc/pihole/list.* 2> /dev/null || true
  538. echo -e "${OVER} ${TICK} ${str}"
  539. fi
  540. detect_pihole_blocking_status
  541. # Determine which functions to run
  542. if [[ "${skipDownload}" == false ]]; then
  543. # Gravity needs to download blocklists
  544. gravity_CheckDNSResolutionAvailable
  545. gravity_GetBlocklistUrls
  546. if [[ "${haveSourceUrls}" == true ]]; then
  547. gravity_SetDownloadOptions
  548. fi
  549. gravity_ConsolidateDownloadedBlocklists
  550. gravity_SortAndFilterConsolidatedList
  551. else
  552. # Gravity needs to modify Blacklist/Whitelist/Wildcards
  553. echo -e " ${INFO} Using cached Event Horizon list..."
  554. numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
  555. echo -e " ${INFO} ${COL_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon"
  556. fi
  557. # Perform when downloading blocklists, or modifying the whitelist
  558. if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "whitelist" ]]; then
  559. gravity_Whitelist
  560. fi
  561. convert_wildcard_to_regex
  562. gravity_ShowBlockCount
  563. # Perform when downloading blocklists, or modifying the white/blacklist (not wildcards)
  564. if [[ "${skipDownload}" == false ]] || [[ "${listType}" == *"list" ]]; then
  565. str="Parsing domains into hosts format"
  566. echo -ne " ${INFO} ${str}..."
  567. gravity_ParseUserDomains
  568. # Perform when downloading blocklists
  569. if [[ ! "${listType:-}" == "blacklist" ]]; then
  570. gravity_ParseLocalDomains
  571. gravity_ParseBlacklistDomains
  572. fi
  573. echo -e "${OVER} ${TICK} ${str}"
  574. gravity_Cleanup
  575. fi
  576. echo ""
  577. # Determine if DNS has been restarted by this instance of gravity
  578. if [[ -z "${dnsWasOffline:-}" ]]; then
  579. # Use "force-reload" when restarting dnsmasq for everything but Wildcards
  580. "${PIHOLE_COMMAND}" restartdns "${dnsRestartType:-force-reload}"
  581. fi
  582. "${PIHOLE_COMMAND}" status