4get-mirror/lib/frontend.php

<?php

class frontend{
	
	public function validateurl($url, $net_validate = false){
		
		$url_parts = parse_url($url);
		
		// check if required parts are there
		if(
			!isset($url_parts["scheme"]) ||
			!(
				$url_parts["scheme"] == "http" ||
				$url_parts["scheme"] == "https"
			) ||
			!isset($url_parts["host"])
		){
			return false;
		}
		
		if($net_validate){
			$ip = 
				str_replace(
					["[", "]"], // handle ipv6
					"",
					$url_parts["host"]
				);
			
			// if its not an IP
			if(!filter_var($ip, FILTER_VALIDATE_IP)){
				
				// resolve domain's IP
				$ip = gethostbyname($url_parts["host"] . ".");
			}
			
			// check if its localhost
			if(
				filter_var(
					$ip,
					FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE
				) === false
			){
				
				return false;
			}
		}
		
		return true;
	}
	
	public function load($template, $replacements = []){
		
		$replacements["server_name"] = htmlspecialchars(config::SERVER_NAME);
		$replacements["version"] = config::VERSION;
		
		if(isset($_COOKIE["theme"])){
			
			$theme = str_replace(["/". "."], "", $_COOKIE["theme"]);
			
			if(
				$theme != "Dark" &&
				!is_file("static/themes/" . $theme . ".css")
			){
				
				$theme = config::DEFAULT_THEME;
			}
		}else{
			
			$theme = config::DEFAULT_THEME;
		}
		
		if($theme != "Dark"){
			
			$replacements["style"] = '<link rel="stylesheet" href="/static/themes/' . rawurlencode($theme) . '.css?v' . config::VERSION . '">';
		}else{
			
			$replacements["style"] = "";
		}
		
		if(isset($_COOKIE["scraper_ac"])){
			
			$replacements["ac"] = '?ac=' . htmlspecialchars($_COOKIE["scraper_ac"]);
		}else{
			
			$replacements["ac"] = '';
		}
		
		if(
			isset($replacements["timetaken"]) &&
			$replacements["timetaken"] !== null
		){
			
			$replacements["timetaken"] = '<div class="timetaken">Took ' . number_format(microtime(true) - $replacements["timetaken"], 2) . 's</div>';
		}
		
		$handle = fopen("template/{$template}", "r");
		$data = fread($handle, filesize("template/{$template}"));
		fclose($handle);
		
		$data = explode("\n", $data);
		$html = "";
		
		for($i=0; $i<count($data); $i++){
			
			$html .= trim($data[$i]);
		}
		
		foreach($replacements as $key => $value){
		
			$html =
				str_replace(
					"{%{$key}%}",
					$value,
					$html
				);
		}
		
		return trim($html);
	}
	
	public function loadheader(array $get, array $filters, string $page, bool $increment_bot_counter = true){
		
		echo
			$this->load("header.html", [
				"title" => trim(htmlspecialchars($get["s"]) . " ({$page})"),
				"description" => ucfirst($page) . ' search results for &quot;' . htmlspecialchars($get["s"]) . '&quot;',
				"index" => "no",
				"search" => htmlspecialchars($get["s"]),
				"tabs" => $this->generatehtmltabs($page, $get["s"]),
				"filters" => $this->generatehtmlfilters($filters, $get)
			]);
		
		$headers_raw = getallheaders();
		$header_keys = [];
		$user_agent = "";
		$bad_header = false;
		
		// block bots that present X-Forwarded-For, Via, etc
		foreach($headers_raw as $headerkey => $headervalue){
			
			$headerkey = strtolower($headerkey);
			if($headerkey == "user-agent"){
				
				$user_agent = $headervalue;
				continue;
			}
			
			// check header key
			if(in_array($headerkey, config::FILTERED_HEADER_KEYS)){
				
				$bad_header = true;
				break;
			}
		}
		
		// SSL check
		$bad_ssl = false;
		if(
			isset($_SERVER["https"]) &&
			$_SERVER["https"] == "on" &&
			isset($_SERVER["SSL_CIPHER"]) &&
			in_array($_SERVER["SSL_CIPHER"], config::FILTERED_HEADER_KEYS)
		){
			
			$bad_ssl = true;
		}
		
		if(
			$bad_header === true ||
			$bad_ssl === true ||
			$user_agent == "" ||
			// user agent check
			preg_match(
				config::HEADER_REGEX,
				$user_agent
			)
		){
			
			// bot detected !!
			if($increment_bot_counter){
				
				apcu_inc(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800);
			}
			
			$this->drawerror(
				"Tshh, blocked!",
				'Your browser, IP or IP range has been blocked from this 4get instance. If this is an error, please <a href="/about">contact the administrator</a>.'
			);
		}
	}
	
	public function drawerror($title, $error, $timetaken = null){
		
		if($timetaken === null){
			
			$timetaken = microtime(true);
		}
		
		echo
			$this->load("search.html", [
				"timetaken" => $timetaken,
				"class" => "",
				"right-left" => "",
				"right-right" => "",
				"left" =>
					'<div class="infobox">' .
						'<h1>' . htmlspecialchars($title) . '</h1>' .
						$error .
					'</div>'
			]);
		die();
	}
	
	public function drawscrapererror($error, $get, $target, $timetaken = null){
		
		if($timetaken === null){
			
			$timetaken = microtime(true);
		}
		
		$this->drawerror(
			"Shit",
			'This scraper returned an error:' .
			'<div class="code">' . htmlspecialchars($error) . '</div>' .
			'Things you can try:' .
			'<ul>' .
				'<li>Use a different scraper</li>' .
				'<li>Remove keywords that could cause errors</li>' .
				'<li><a href="/instances?target=' . $target . "&" . $this->buildquery($get, false) . '">Try your search on another 4get instance</a></li>' .
			'</ul><br>' .
			'If the error persists, please <a href="/about">contact the administrator</a>.',
			$timetaken
		);
	}
	
	public function drawtextresult($site, $greentext = null, $duration = null, $keywords, $tabindex = true, $customhtml = null){
		
		$payload =
			'<div class="text-result">';
		
		// add favicon, link and archive links
		$payload .= $this->drawlink($site["url"]);
		
		/*
			Draw title + description + filetype
		*/
		$payload .=
			'<a href="' . htmlspecialchars($site["url"]) . '" class="hover" rel="noreferrer nofollow"';
			
		if($tabindex === false){
			
			$payload .= ' tabindex="-1"';
		}
			
		$payload .= '>';
			
			if($site["thumb"]["url"] !== null){
				
				$payload .=
					'<div class="thumb-wrap';
				
				switch($site["thumb"]["ratio"]){
					
					case "16:9":
						$size = "landscape";
						break;
					
					case "9:16":
						$payload .= " portrait";
						$size = "portrait";
						break;
					
					case "1:1":
						$payload .= " square";
						$size = "square";
						break;
				}
				
				$payload .=
					'">' .
						'<img class="thumb" src="' . $this->htmlimage($site["thumb"]["url"], $size) . '" alt="thumb">';
				
				if($duration !== null){
					
					$payload .=
						'<div class="duration">' .
							htmlspecialchars($duration) .
						'</div>';
				}
				
				$payload .=
					'</div>';
			}
			
		$payload .=
			'<div class="title">';
		
		if(
			isset($site["type"]) &&
			$site["type"] != "web"
		){
			
			$payload .= '<div class="type">' . strtoupper($site["type"]) . '</div>';
		}
		
		$payload .=
			$this->highlighttext($keywords, $site["title"]) .
		'</div>';
		
		if($greentext !== null){
			
			$payload .=
				'<div class="greentext">' .
					htmlspecialchars($greentext) .
				'</div>';
		}
		
		if($site["description"] !== null){
			
			$payload .=
				'<div class="description">' .
					$this->highlighttext($keywords, $site["description"]) .
				'</div>';
		}
		
		$payload .= $customhtml;
		
		$payload .= '</a>';
		
		/*
			Sublinks
		*/
		if(
			isset($site["sublink"]) &&
			!empty($site["sublink"])
		){
			
			usort($site["sublink"], function($a, $b){
				
				return strlen($a["description"]) > strlen($b["description"]);
			});
			
			$payload .=
				'<div class="sublinks">' .
					'<table>';
			
			$opentr = false;
			for($i=0; $i<count($site["sublink"]); $i++){
				
				if(($i % 2) === 0){
					
					$opentr = true;
					$payload .= '<tr>';
				}else{
					
					$opentr = false;
				}
				
				$payload .=
					'<td>' .
						'<a href="' . htmlspecialchars($site["sublink"][$i]["url"]) . '" rel="noreferrer nofollow">' .
							'<div class="title">' .
								htmlspecialchars($site["sublink"][$i]["title"]) .
							'</div>';
				
				if(!empty($site["sublink"][$i]["date"])){
					
					$payload .=
						'<div class="greentext">' .
							date("jS M y @ g:ia", $site["sublink"][$i]["date"]) .
						'</div>';
				}
				
				if(!empty($site["sublink"][$i]["description"])){
					
					$payload .=
						'<div class="description">' .
							$this->highlighttext($keywords, $site["sublink"][$i]["description"]) .
						'</div>';
				}
				
				$payload .= '</a></td>';
				
				if($opentr === false){
					
					$payload .= '</tr>';
				}
			}
			
			if($opentr === true){
				
				$payload .= '<td></td></tr>';
			}
			
			$payload .= '</table></div>';
		}
		
		if(
			isset($site["table"]) &&
			!empty($site["table"])
		){
			
			$payload .= '<table class="info-table">';
			
			foreach($site["table"] as $title => $value){
				
				$payload .=
					'<tr>' .
						'<td>' . htmlspecialchars($title) . '</td>' .
						'<td>' . htmlspecialchars($value) . '</td>' .
					'</tr>';
			}
			
			$payload .= '</table>';
		}
		
		return $payload . '</div>';
	}
	
	public function highlighttext($keywords, $text){
		
		$text = htmlspecialchars($text);
		
		$keywords = explode(" ", $keywords);
		$regex = [];
		
		foreach($keywords as $word){
			
			$regex[] = "\b" . preg_quote($word, "/") . "\b";
		}
		
		$regex = "/" . implode("|", $regex) . "/i";
		
		return
			preg_replace(
				$regex,
				'<b>${0}</b>',
				$text
			);
	}
	
	function highlightcode($text){
		
		// https://www.php.net/highlight_string
		ini_set("highlight.comment", "c-comment");
		ini_set("highlight.default", "c-default");
		ini_set("highlight.html", "c-default");
		ini_set("highlight.keyword", "c-keyword");
		ini_set("highlight.string", "c-string");
		
		$text =
			trim(
				preg_replace(
					'/<code [^>]+>/',
					"",
					str_replace(
						[
							"<br />",
							"&nbsp;",
							"<pre>",
							"</pre>",
							"</code>"
						],
						[
							"\n",
							" ",
							"",
							"",
							""
						],
						explode(
							"&lt;?php",
							highlight_string("<?php " . $text, true),
							2
						)[1]
					)
				)
			);
		
		// replace colors
		$classes = ["c-comment", "c-default", "c-keyword", "c-string"];
		
		foreach($classes as $class){
			
			$text = str_replace('<span style="color: ' . $class . '">', '<span class="' . $class . '">', $text);
		}
		
		return $text;
	}
	
	public function drawlink($link){
		
		/*
			Add favicon
		*/
		$host = parse_url($link);
		
		// special case for when we're not drawing a full url
		if(!isset($host["host"])){
			
			$payload =
				'<div class="url">' .
					'<button class="favicon" tabindex="-1">' .
						'<img src="/favicon?s=404" alt="xx">' .
					'</button>';
		}else{
			
			$esc =
				explode(
					".",
					$host["host"],
					2
				);
			
			if(
				count($esc) === 2 &&
				$esc[0] == "www"
			){
				
				$esc = $esc[1];
			}else{
				
				$esc = $esc[0];
			}
			
			$esc = substr($esc, 0, 2);
			
			$urlencode = urlencode($link);
			
			$payload =
				'<div class="url">' .
					'<button class="favicon" tabindex="-1">' .
						'<img src="/favicon?s=' . htmlspecialchars($host["scheme"] . "://" . $host["host"]) . '" alt="' . htmlspecialchars($esc) . '">' .
						//'<img src="/404.php" alt="' . htmlspecialchars($esc) . '">' .
					'</button>' .
					'<div class="favicon-dropdown">';
			
			$archives = [];
			
			// add website-specific archives
			switch($host["host"]){
				
				case "www.youtube.com":
				case "music.youtube.com":
				case "youtube.com":
				case "m.youtube.com":
				case "youtu.be":
					
					if(
						(
							$host["host"] == "youtu.be" &&
							isset($host["path"]) &&
							preg_match(
								'/^\/([A-Za-z0-9_-]+)/',
								$host["path"],
								$slug
							)
						) ||
						(
							isset($host["query"]) &&
							preg_match(
								'/v=([A-Za-z0-9_-]+)/',
								$host["query"],
								$slug
							)
						)
					){
						
						// for watch?v=slug
						$archives[] = [
							"url" => "https://findyoutubevideo.thetechrobo.ca/?q=" . $slug[1],
							"favicon" => "findyoutubevideo.thetechrobo.ca",
							"favicon_alt" => "fi",
							"title" => "FindYouTubeVideo"
						];
						
						$archives[] = [
							"url" => "https://filmot.com/video/" . $slug[1] . "/",
							"favicon" => "filmot.com",
							"favicon_alt" => "fi",
							"title" => "Filmot (metadata only)"
						];
					}elseif(
						preg_match(
							'/^\/(?:channel)\/@?([A-Za-z0-9_-]+)/',
							$host["path"],
							$username
						)
					){
						
						$archives[] = [
							"url" => "https://filmot.com/channel/" . $username[1] . "/",
							"favicon" => "filmot.com",
							"favicon_alt" => "fi",
							"title" => "Filmot (metadata only)"
						];
					}
					break;
				
				case "twitch.tv":
				case "player.twitch.tv":
				case "www.twitch.tv":
					if(
						isset($host["path"]) &&
						preg_match(
							'/^\/([A-Za-z0-9_.]+)/',
							$host["path"],
							$username
						) &&
						$username[1] != "videos"
					){
						
						$archives[] = [
							"url" => "https://vodvod.top/channels/@" . $username[1],
							"favicon" => "vodvod.top",
							"favicon_alt" => "vo",
							"title" => "vodvod"
						];
						
						$archives[] = [
							"url" => "https://twitchtracker.com/" . $username[1],
							"favicon" => "twitchtracker.com",
							"favicon_alt" => "tw",
							"title" => "TwitchTracker"
						];
					}
					
					break;
				
				case "kick.com":
				case "player.kick.com":
					if(
						isset($host["path"]) &&
						preg_match(
							'/^\/([A-Za-z0-9_.]+)/',
							$host["path"],
							$username
						)
					){
						
						$archives[] = [
							"url" => "https://lick.lolcat.ca/channel?name=" . $username[1],
							"favicon" => "lick.lolcat.ca",
							"favicon_alt" => "li",
							"title" => "lick"
						];
						
						$archives[] = [
							"url" => "https://kicktracker.net/" . $username[1],
							"favicon" => "kicktracker.net",
							"favicon_alt" => "ki",
							"title" => "Kick tracker"
						];
					}
					break;
				
				case "x.com":
				case "twitter.com":
					if(
						isset($host["path"]) &&
						preg_match(
							'/^\/([A-Za-z0-9_.]+)/',
							$host["path"],
							$username
						)
					){
						
						$archives[] = [
							"url" => "https://web.archive.org/web/*/https://x.com/" . $username[1] . "/status*",
							"favicon" => "archive.org",
							"favicon_alt" => "ar",
							"title" => "Archive.org: Tweets &gt;2023"
						];
						
						$archives[] = [
							"url" => "https://web.archive.org/web/*/https://twitter.com/" . $username[1] . "/status*",
							"favicon" => "archive.org",
							"favicon_alt" => "ar",
							"title" => "Archive.org: Tweets &lt;2023"
						];
					}
					break;
				
				case "www.instagram.com":
				case "instagram.com":
					if(
						isset($host["path"]) &&
						preg_match(
							'/^\/([A-Za-z0-9_.]+)/',
							$host["path"],
							$username
						) &&
						$username[1] != "p"
					){
						
						$archives[] = [
							"url" => "https://instarchiver.net/users?q=" . $username[1],
							"favicon" => "instarchiver.net",
							"favicon_alt" => "in",
							"title" => "Instarchiver"
						];
						
						$archives[] = [
							"url" => "https://www.storiesdb.ch/" . $username[1],
							"favicon" => "www.storiesdb.ch",
							"favicon_alt" => "st",
							"title" => "StoriesDB"
						];
					}
					break;
				
				case "reddit.com":
				case "old.reddit.com":
				case "www.reddit.com":
					if(isset($host["path"])){
						// direct thread lookup
						// https://ihsoyct.github.io/r/selfhosted/comments/16emfv0/4get_a_proxy_search_engine_that_doesnt_suck/?backend=pullpush
						// https://ihsoyct.github.io/r/selfhosted/comments/16emfv0/4get_a_proxy_search_engine_that_doesnt_suck/?backend=artic_shift
						if(
							preg_match(
								'/^\/r\/[^\/]+\/comments\/[^?&]+/',
								$host["path"],
								$slug
							)
						){
							
							$archives[] = [
								"url" => "https://ihsoyct.github.io{$slug[0]}?backend=artic_shift",
								"favicon" => "reddit.com",
								"favicon_alt" => "re",
								"title" => "Artic Shift"
							];
							
							$archives[] = [
								"url" => "https://ihsoyct.github.io{$slug[0]}?backend=pullpush",
								"favicon" => "reddit.com",
								"favicon_alt" => "re",
								"title" => "PullPush"
							];
						}
						
						// subreddit thread search
						// https://ihsoyct.github.io/?subreddit=selfhosted&backend=artic_shift&mode=submissions&sort=desc
						// https://ihsoyct.github.io/?subreddit=selfhosted&backend=pullpush&mode=submissions&sort=desc
						// subreddit comment search
						// https://ihsoyct.github.io/?subreddit=selfhosted&backend=artic_shift&mode=comments&sort=desc
						// https://ihsoyct.github.io/?subreddit=selfhosted&backend=pullpush&mode=comments&sort=desc
						elseif(
							preg_match(
								'/^\/r\/([^\/]+)\/(?:$|\?|&|search|wiki)/',
								$host["path"],
								$slug
							)
						){
							
							$archives[] = [
								"url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=artic_shift&mode=submissions&sort=desc",
								"favicon" => "reddit.com",
								"favicon_alt" => "re",
								"title" => "Artic Shift (Search threads)"
							];
							
							$archives[] = [
								"url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=pullpush&mode=submissions&sort=desc",
								"favicon" => "reddit.com",
								"favicon_alt" => "re",
								"title" => "PullPush (Search threads)"
							];
							
							$archives[] = [
								"url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=artic_shift&mode=comments&sort=desc",
								"favicon" => "reddit.com",
								"favicon_alt" => "re",
								"title" => "Artic Shift (Search comments)"
							];
							
							$archives[] = [
								"url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=pullpush&mode=comments&sort=desc",
								"favicon" => "reddit.com",
								"favicon_alt" => "re",
								"title" => "PullPush (Search comments)"
							];
						}
						
						// user thread lookup
						// https://ihsoyct.github.io/index.html?author=google&mode=submissions&backend=artic_shift
						// https://ihsoyct.github.io/index.html?author=google&mode=submissions&backend=pullpush
						// user comment lookup
						// https://ihsoyct.github.io/index.html?author=google&mode=comments&backend=artic_shift
						// https://ihsoyct.github.io/index.html?author=google&mode=comments&backend=pullpush
						elseif(
							preg_match(
								'/^\/(?:u|user)\/([^\/]+)\//',
								$host["path"],
								$slug
							)
						){
							
							$archives[] = [
								"url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=submissions&backend=artic_shift",
								"favicon" => "reddit.com",
								"favicon_alt" => "re",
								"title" => "Artic Shift (Search threads)"
							];
							
							$archives[] = [
								"url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=submissions&backend=pullpush",
								"favicon" => "reddit.com",
								"favicon_alt" => "re",
								"title" => "PullPush (Search threads)"
							];
							
							$archives[] = [
								"url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=comments&backend=artic_shift",
								"favicon" => "reddit.com",
								"favicon_alt" => "re",
								"title" => "Artic Shift (Search comments)"
							];
							
							$archives[] = [
								"url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=comments&backend=pullpush",
								"favicon" => "reddit.com",
								"favicon_alt" => "re",
								"title" => "PullPush (Search comments)"
							];
						}
					}
					break;
			}
			
			// detect git
			if(
				(
					stripos(
						$host["host"],
						"git."
					) !== false ||
					$host["host"] == "codeberg.org" ||
					$host["host"] == "sourceforge.net" ||
					$host["host"] == "github.com"
				) &&
				isset($host["path"]) &&
				preg_match(
					'/^(\/[^\/]+\/[^\/]+\/?)/',
					$host["path"],
					$edge
				)
			){
				
				$archives[] = [
					"url" => "https://archive.softwareheritage.org/browse/origin/directory/?origin_url=" . urlencode($host["scheme"] . "://" . $host["host"] . $edge[1]),
					"favicon" => "www.softwareheritage.org",
					"favicon_alt" => "so",
					"title" => "SoftwareHeritage"
				];
			}
			
			$archives =
				array_merge(
					$archives,
					[
						[
							"url" => "https://web.archive.org/web/" . $urlencode,
							"favicon" => "archive.org",
							"favicon_alt" => "ar",
							"title" => "Archive.org"
						],
						[
							"url" => "https://archive.ph/newest/" . htmlspecialchars($link),
							"favicon" => "archive.ph",
							"favicon_alt" => "ar",
							"title" => "Archive.ph"
						],
						[
							"url" => "https://yandex.com/search/?text=" . urlencode("url:" . $link),
							"favicon" => "yandex.com",
							"favicon_alt" => "ya",
							"title" => "Yandex cache"
						],
						[
							"url" => "https://ghostarchive.org/search?term=" . $urlencode,
							"favicon" => "ghostarchive.org",
							"favicon_alt" => "gh",
							"title" => "Ghostarchive"
						],
						[
							"url" => "https://arquivo.pt/wayback/" . htmlspecialchars($link),
							"favicon" => "arquivo.pt",
							"favicon_alt" => "ar",
							"title" => "Arquivo.pt"
						],
						[
							"url" => "https://megalodon.jp/?url=" . $urlencode,
							"favicon" => "megalodon.jp",
							"favicon_alt" => "me",
							"title" => "Megalodon"
						],
						[
							"url" => "https://www.webcitation.org/query?url=" . $urlencode,
							"favicon" => "webcitation.org",
							"favicon_alt" => "we",
							"title" => "Webcitation"
						]
					]
				);
			
			foreach($archives as $archive){
				
				$payload .= '<a href="' . $archive["url"] . '" class="list" target="_BLANK"><img src="/favicon?s=https://' . $archive["favicon"] . '" alt="' . $archive["favicon_alt"] . '">' . $archive["title"] . '</a>';
			}
			
			$payload .=	'</div>';
		}
		
		/*
			Draw link
		*/
		$parts = explode("/", $link);
		$clickurl = "";
		
		// remove trailing /
		$c = count($parts) - 1;
		if($parts[$c] == ""){
			
			$parts[$c - 1] = $parts[$c - 1] . "/";
			unset($parts[$c]);
		}
		
		// merge https://site together
		if(isset($host["host"])){
			$parts = [
				$parts[0] . $parts[1] . '//' . $parts[2],
				...array_slice($parts, 3, count($parts) - 1)
			];
		}
		
		$c = count($parts);
		for($i=0; $i<$c; $i++){
			
			if($i !== 0){ $clickurl .= "/"; }
			
			$clickurl .= $parts[$i];
			
			if($i === $c - 1){
				
				$parts[$i] = rtrim($parts[$i], "/");
			}
			
			$payload .=
				'<a class="part" href="' . htmlspecialchars($clickurl) . '" rel="noreferrer nofollow" tabindex="-1">' .
					htmlspecialchars(urldecode($parts[$i])) .
				'</a>';
			
			if($i !== $c - 1){
				
				$payload .= '<span class="separator"></span>';
			}
		}
		
		return $payload . '</div>';
	}
	
	public function getscraperfilters($page){
		
		// hack: enable error reporting if configured
		if(config::DISPLAY_ERRORS === true){
			
			ini_set('display_errors', 1);
			ini_set('display_startup_errors', 1);
		}
		
		$get_scraper = isset($_COOKIE["scraper_$page"]) ? $_COOKIE["scraper_$page"] : null;
		
		if(
			isset($_GET["scraper"]) &&
			is_string($_GET["scraper"])
		){
			
			$get_scraper = $_GET["scraper"];
		}else{
			
			if(
				isset($_GET["npt"]) &&
				is_string($_GET["npt"])
			){
				
				$get_scraper = explode(".", $_GET["npt"], 2)[0];
				
				$get_scraper =
					preg_replace(
						'/[0-9]+$/',
						"",
						$get_scraper
					);
			}
		}
		
		// add search field
		$filters =
			[
				"s" => [
					"option" => "_SEARCH"
				]
			];
		
		// define default scrapers
		switch($page){
			
			case "web":
				$filters["scraper"] = [
					"display" => "Scraper",
					"option" => [
						//"fget" => "fget",
						"ddg" => "DuckDuckGo",
						//"yahoo" => "Yahoo!",
						"brave" => "Brave",
						"yandex" => "Yandex",
						"google" => "Google",
						"google_api" => "Google API",
						"google_cse" => "Google CSE",
						"yahoo_japan" => "Yahoo! JAPAN",
						"startpage" => "Startpage",
						"yep" => "Yep",
						"mwmbl" => "Mwmbl",
						"mojeek" => "Mojeek",
						"naver" => "Naver",
						"baidu" => "Baidu",
						"coccoc" => "Cốc Cốc",
						"solofield" => "Solofield",
						"marginalia" => "Marginalia",
						"purili" => "Purili",
						"wiby" => "wiby"
					]
				];
				break;
			
			case "images":
				$filters["scraper"] = [
					"display" => "Scraper",
					"option" => [
						"ddg" => "DuckDuckGo",
						"yandex" => "Yandex",
						"brave" => "Brave",
						"google" => "Google",
						"google_api" => "Google API",
						"google_cse" => "Google CSE",
						"yahoo_japan" => "Yahoo! JAPAN",
						"startpage" => "Startpage",
						"naver" => "Naver",
						"baidu" => "Baidu",
						"solofield" => "Solofield",
						"pinterest" => "Pinterest",
						"flickr" => "Flickr",
						"pexels" => "Pexels",
						"pixabay" => "Pixabay",
						"unsplash" => "Unsplash",
						"fivehpx" => "500px",
						"vsco" => "VSCO",
						"imgur" => "Imgur",
						"ftm" => "FindThatMeme"
					]
				];
				break;
			
			case "videos":
				$filters["scraper"] = [
					"display" => "Scraper",
					"option" => [
						"yt" => "YouTube",
						//"archiveorg" => "Archive.org",
						//"dailymotion" => "Dailymotion",
						"vimeo" => "Vimeo",
						//"odysee" => "Odysee",
						"sepiasearch" => "Sepia Search",
						//"fb" => "Facebook videos",
						"ddg" => "DuckDuckGo",
						"brave" => "Brave",
						"yandex" => "Yandex",
						"google" => "Google",
						"yahoo_japan" => "Yahoo! JAPAN",
						"startpage" => "Startpage",
						"naver" => "Naver",
						"baidu" => "Baidu",
						"coccoc" => "Cốc Cốc",
						"purili" => "Purili",
						"solofield" => "Solofield"
					]
				];
				break;
			
			case "news":
				$filters["scraper"] = [
					"display" => "Scraper",
					"option" => [
						"ddg" => "DuckDuckGo",
						"brave" => "Brave",
						"google" => "Google",
						"yahoo_japan" => "Yahoo! JAPAN",
						"startpage" => "Startpage",
						//"mojeek" => "Mojeek",
						"baidu" => "Baidu"
					]
				];
				break;
			
			case "music":
				$filters["scraper"] = [
					"display" => "Scraper",
					"option" => [
						"sc" => "SoundCloud",
						"swisscows" => "Swisscows (SoundCloud)"
						//"spotify" => "Spotify"
					]
				];
				break;
			
			case "booru":
				$filters["scraper"] = [
					"display" => "Scraper",
					"option" => [
						"safebooru" => "Safebooru",
						"konachan" => "Konachan",
						"tbib" => "The Big Imageboard",
						"gelbooru" => "Gelbooru",
						"yandere" => "Yande.re",
						"tbib" => "The Big Imageboard",
						"sankakucomplex" => "SankakuComplex",
						"soybooru" => "SoyBooru"
					]
				];
				break;
		}
		
		// get scraper name from user input, or default out to preferred scraper
		$scraper_out = null;
		$first = true;
		
		foreach($filters["scraper"]["option"] as $scraper_name => $scraper_pretty){
			
			if($first === true){
				
				$first = $scraper_name;
			}
			
			if($scraper_name == $get_scraper){
				
				$scraper_out = $scraper_name;
			}
		}
		
		if($scraper_out === null){
			
			$scraper_out = $first;
		}
		
		include "scraper/$scraper_out.php";
		$lib = new $scraper_out();
		
		// set scraper on $_GET
		$_GET["scraper"] = $scraper_out;
		
		// set nsfw on $_GET
		if(
			isset($_COOKIE["nsfw"]) &&
			!isset($_GET["nsfw"])
		){
			
			$_GET["nsfw"] = $_COOKIE["nsfw"];
		}
		
		return
			[
				$lib,
				array_merge_recursive(
					$filters,
					$lib->getfilters($page)
				)
			];
	}
	
	public function parsegetfilters($parameters, $whitelist){
		
		$sanitized = [];
		
		// add npt token
		if(
			isset($parameters["npt"]) &&
			is_string($parameters["npt"])
		){
			
			$sanitized["npt"] = $parameters["npt"];
		}else{
			
			$sanitized["npt"] = false;
		}
		
		// we're iterating over $whitelist, so
		// you can't polluate $sanitized with useless
		// parameters
		foreach($whitelist as $parameter => $value){
			
			if(isset($parameters[$parameter])){
				
				if(!is_string($parameters[$parameter])){
					
					$sanitized[$parameter] = null;
					continue;
				}
				
				// parameter is already set, use that value
				$sanitized[$parameter] = $parameters[$parameter];
			}else{
				
				// parameter is not set, add it
				if(is_string($value["option"])){
					
					// special field: set default value manually
					switch($value["option"]){
						
						case "_DATE":
							// no date set
							$sanitized[$parameter] = false;
							break;
						
						case "_SEARCH":
							// no search set
							$sanitized[$parameter] = "";
							break;
					}
					
				}else{
					
					// set a default value
					$sanitized[$parameter] = array_keys($value["option"])[0];
				}
			}
			
			// sanitize input
			if(is_array($value["option"])){
				if(
					!in_array(
						$sanitized[$parameter],
						$keys = array_keys($value["option"])
					)
				){
					
					$sanitized[$parameter] = $keys[0];
				}
			}else{
				
				// sanitize search & string
				switch($value["option"]){
					
					case "_DATE":
						if($sanitized[$parameter] !== false){
							
							$sanitized[$parameter] = strtotime($sanitized[$parameter]);
							if($sanitized[$parameter] <= 0){
								
								$sanitized[$parameter] = false;
							}
						}
						break;
					
					case "_SEARCH":
						// get search string
						$sanitized["s"] = trim($sanitized[$parameter]);
				}
			}
		}
		
		// invert dates if needed
		if(
			isset($sanitized["older"]) &&
			isset($sanitized["newer"]) &&
			$sanitized["newer"] !== false &&
			$sanitized["older"] !== false &&
			$sanitized["newer"] > $sanitized["older"]
		){
			
			// invert
			[
				$sanitized["older"],
				$sanitized["newer"]
			] = [
				$sanitized["newer"],
				$sanitized["older"]
			];
		}
		
		return $sanitized;
	}

	public function s_to_timestamp($seconds){
		
		if(is_string($seconds)){
			
			return "LIVE";
		}
		
		return ($seconds >= 60) ? ltrim(gmdate("H:i:s", $seconds), ":0") : gmdate("0:s", $seconds);
	}
	
	public function generatehtmltabs($page, $query){
		
		$html = null;
		
		//foreach(["web", "images", "videos", "news", "music", "booru"] as $type){
		foreach(["web", "images", "videos", "news", "music"] as $type){
			
			$html .= '<a href="/' . $type . '?s=' . urlencode($query);
			
			if(!empty($params)){
				
				$html .= $params;
			}
			
			$html .= '" class="tab';
			
			if($type == $page){
				
				$html .= ' selected';
			}
			
			$html .= '">' . ucfirst($type) . '</a>';
		}
		
		return $html;
	}
	
	public function generatehtmlfilters($filters, $params){
		
		$html = null;
		
		foreach($filters as $filter_name => $filter_values){
			
			if(!isset($filter_values["display"])){
				
				continue;
			}
			
			$output = true;
			$tmp =
				'<div class="filter">' .
					'<div class="title">' . htmlspecialchars($filter_values["display"]) . '</div>';
			
			if(is_array($filter_values["option"])){
				
				$tmp .= '<select name="' . $filter_name . '">';
				
				foreach($filter_values["option"] as $option_name => $option_title){
					
					$tmp .= '<option value="' . $option_name . '"';
					
					if($params[$filter_name] == $option_name){
						
						$tmp .= ' selected';
					}
					
					$tmp .= '>' . htmlspecialchars($option_title) . '</option>';
				}
				
				$tmp .= '</select>';
			}else{
				
				switch($filter_values["option"]){
					
					case "_DATE":
						$tmp .= '<input type="date" name="' . $filter_name . '"';
						
						if($params[$filter_name] !== false){
							
							$tmp .= ' value="' . date("Y-m-d", $params[$filter_name]) . '"';
						}
						
						$tmp .= '>';
						break;
					
					default:
						$output = false;
						break;
				}
			}
			
			$tmp .= '</div>';
			
			if($output === true){
				
				$html .= $tmp;
			}
		}
		
		return $html;
	}
	
	public function buildquery($gets, $ommit = false){
		
		$out = [];
		foreach($gets as $key => $value){
			
			if(
				$value == null ||
				$value == false ||
				$key == "npt" ||
				$key == "extendedsearch" ||
				$value == "any" ||
				$value == "all" ||
				$key == "spellcheck" ||
				(
					$ommit === true &&
					$key == "s"
				)
			){
				
				continue;
			}
			
			if(
				$key == "older" ||
				$key == "newer"
			){
				
				$value = date("Y-m-d", (int)$value);
			}
			
			$out[$key] = $value;
		}
		
		return http_build_query($out);
	}
	
	public function increment_real_reqs($scraper){
		
		apcu_inc(intdiv(time(), 3600) . ".real_requests", 1, $s, 262800);
		apcu_inc(intdiv(time(), 3600) . ".$scraper.requests", 1, $s, 262800);
	}
	
	public function htmlimage($image, $format){
		
		if(
			preg_match(
				'/^data:/',
				$image
			)
		){
			
			return htmlspecialchars($image);
		}
		
		//return "https://4get.ca/proxy?i=" . urlencode($image) . "&s=" . $format;
		return "/proxy?i=" . urlencode($image) . "&s=" . $format;
	}
	
	public function htmlnextpage($gets, $npt, $page){
		
		$query = $this->buildquery($gets);
		
		return $page . "?" . $query . "&npt=" . $npt;
	}
}