4get-mirror/scraper/startpage.php

<?php

class startpage{
	
	public function __construct(){
		
		include "lib/backend.php";
		$this->backend = new backend("startpage");
		
		include "lib/fuckhtml.php";
		$this->fuckhtml = new fuckhtml();
		
		include "lib/anubis.php";
		$this->anubis = new anubis();
		$this->anubis_error = "all gucci";
	}
	
	public function getfilters($page){
		
		switch($page){
			case "web":
				return [
					"country" => [
						"display" => "Country",
						"option" => [
							"any" => "All Regions",
							"es_AR" => "Argentina",
							"en_AU" => "Australia",
							"de_AT" => "Austria",
							"ru_BY" => "Belarus",
							"fr_BE" => "Belgium (FR)",
							"nl_BE" => "Belgium (NL)",
							"bg_BG" => "Bulgaria",
							"en_CA" => "Canada (EN)",
							"fr_CA" => "Canada (FR)",
							"es_CL" => "Chile",
							"es_CO" => "Colombia",
							"cs_CZ" => "Czech Republic",
							"da_DK" => "Denmark",
							"ar_EG" => "Egypt",
							"et_EE" => "Estonia",
							"fi_FI" => "Finland",
							"fr_FR" => "France",
							"de_DE" => "Germany",
							"el_GR" => "Greece",
							"hu_HU" => "Hungary",
							"hi_IN" => "India (HI)",
							"en_IN" => "India (EN)",
							"id_ID" => "Indonesia (ID)",
							"en_ID" => "Indonesia (EN)",
							"en_IE" => "Ireland",
							"it_IT" => "Italy",
							"ja_JP" => "Japan",
							"ko_KR" => "Korea",
							"ms_MY" => "Malaysia (MS)",
							"en_MY" => "Malaysia (EN)",
							"es_MX" => "Mexico",
							"nl_NL" => "Netherlands",
							"en_NZ" => "New Zealand",
							"no_NO" => "Norway",
							"es_PE" => "Peru",
							"fil_PH" => "Philippines (FIL)",
							"en_PH" => "Philippines (EN)",
							"pl_PL" => "Poland",
							"pt_PT" => "Portugal",
							"ro_RO" => "Romania",
							"ru_RU" => "Russia",
							"ms_SG" => "Singapore (MS)",
							"en_SG" => "Singapore (EN)",
							"es_ES" => "Spain (ES)",
							"ca_ES" => "Spain (CA)",
							"sv_SE" => "Sweden",
							"de_CH" => "Switzerland (DE)",
							"fr_CH" => "Switzerland (FR)",
							"it_CH" => "Switzerland (IT)",
							"tr_TR" => "Turkey",
							"uk_UA" => "Ukraine",
							"en_US" => "US (EN)",
							"es_US" => "US (ES)",
							"es_UY" => "Uruguay",
							"es_VE" => "Venezuela",
							"vi_VN" => "Vietnam (VI)",
							"en_VN" => "Vietnam (EN)",
							"en_ZA" => "South Africa"
						]
					],
					"nsfw" => [ // qadf
						"display" => "NSFW",
						"option" => [
							"yes" => "Yes", // qadf=none
							"no" => "No" // qadf=heavy
						]
					],
					"time" => [ // with_date
						"display" => "Time posted",
						"option" => [
							"any" => "Any time",
							"d" => "Past 24 hours",
							"w" => "Past week",
							"m" => "Past month",
							"y" => "Past year",
						]
					],
					"extendedsearch" => [
						// undefined display, so it wont show in frontend
						"option" => [
							"yes" => "Yes",
							"no" => "No"
						]
					]
				];
				break;
			
			case "images":
				return [
					"nsfw" => [ // qadf
						"display" => "NSFW",
						"option" => [
							"yes" => "Yes", // qadf=none
							"no" => "No" // qadf=heavy
						]
					],
					"size" => [ // flimgsize
						"display" => "Size",
						"option" => [
							"any" => "Any size",
							"Small" => "Small",
							"Medium" => "Medium",
							"Large" => "Large",
							"Wallpaper" => "Wallpaper",
							// from here, image-size-select, var prefix = isz:lt,islt:
							"qsvgs" => "Larger than 400x300",
							"vga" => "Larger than 640x480",
							"svga" => "Larger than 800x600",
							"xga" => "Larger than 1024x768",
							"qsvgs" => "Larger than 400x300",
							"2mp" => "Larger than 2 MP (1600x1200)",
							"4mp" => "Larger than 4 MP (2272x1704)",
							"6mp" => "Larger than 6 MP (2816x2112)",
							"8mp" => "Larger than 8 MP (3264x2448)",
							"10mp" => "Larger than 10 MP (3648x2736)",
							"12mp" => "Larger than 12 MP (4096x3072)",
							"15mp" => "Larger than 15 MP (4480x3360)",
							"20mp" => "Larger than 20 MP (5120x3840)",
							"40mp" => "Larger than 40 MP (7216x5412)",
							"70mp" => "Larger than 70 MP (9600x7200)"
						]
					],
					"color" => [ // flimgcolor
						"display" => "Color",
						"option" => [
							"any" => "Any color",
							// from here, var prefix = ic:
							"color" => "Color only",
							"bnw" => "Black & white", // set to "gray"
							// from here, var prefix = ic:specific,isc:
							"red" => "Red",
							"orange" => "Orange",
							"yellow" => "Yellow",
							"green" => "Green",
							"teal" => "Teal",
							"blue" => "Blue",
							"purple" => "Purple",
							"pink" => "Pink",
							"white" => "White",
							"gray" => "Gray",
							"black" => "Black",
							"brown" => "Brown"
						]
					],
					"type" => [ // flimgtype
						"display" => "Type",
						"option" => [
							"any" => "Any type",
							"AnimatedGif" => "Animated GIF",
							"Clipart" => "Clip Art",
							"Line" => "Line Drawing",
							"Photo" => "Photograph",
							"Transparent" => "Transparent Background"
						]
					],
					"license" => [ // flimglicense
						"display" => "License",
						"option" => [
							"any" => "Any license",
							"p" => "Public domain",
							"s" => "Free to share",
							"sc" => "Free to share commercially",
							"m" => "Free to modify",
							"mc" => "Free to modify commercially"
						]
					]
				];
				break;
			
			case "videos":
				return [
					"nsfw" => [ // qadf
						"display" => "NSFW",
						"option" => [
							"yes" => "Yes", // qadf=none
							"no" => "No" // qadf=heavy
						]
					],
					"sort" => [
						"display" => "Sort by",
						"option" => [
							"relevance" => "Most relevant",
							"popular" => "Most popular",
							"recent" => "Most recent"
						]
					],
					"duration" => [ // with_duration
						"display" => "Duration",
						"option" => [
							"any" => "Any duration",
							"short" => "Short",
							"medium" => "Medium",
							"long" => "Long"
						]
					]
				];
				break;
			
			case "news":
				return [
					"nsfw" => [ // qadf
						"display" => "NSFW",
						"option" => [
							"yes" => "Yes", // qadf=none
							"no" => "No" // qadf=heavy
						]
					],
					"time" => [ // with_date
						"display" => "Time posted",
						"option" => [
							"any" => "Any time",
							"d" => "Past 24 hours",
							"w" => "Past week",
							"m" => "Past month"
						]
					]
				];
				break;
				
				//preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEazerbaijaniN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius; Domain=startpage.com; Expires=Mon, 28 Oct 2024 20:21:58 GMT; Secure; Path=/
				//preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius; Domain=startpage.com; Expires=Mon, 28 Oct 2024 20:22:52 GMT; Secure; Path=/
		}
	}
	
	private function get($proxy, $url, $get = [], $post = false, $is_xhr = false, $return_cookies = false, $passed_cookies = []){
		
		$curlproc = curl_init();
		
		if($post === true){
			
			curl_setopt($curlproc, CURLOPT_POST, true);
			curl_setopt($curlproc, CURLOPT_POSTFIELDS, $get);
			
		}elseif($get !== []){
			
			$get = http_build_query($get);
			$url .= "?" . $get;
		}
		
		if($return_cookies){
			
			$cookies = [];
			curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies){
				
				$length = strlen($header);
				
				$header = explode(":", $header, 2);
				
				if(trim(strtolower($header[0])) == "set-cookie"){
					
					$cookie_tmp = explode("=", trim($header[1]), 2);
					
					$cookies[trim($cookie_tmp[0])] =
						explode(";", $cookie_tmp[1], 2)[0];
				}
				
				return $length;
			});
		}
		
		curl_setopt($curlproc, CURLOPT_URL, $url);
		
		// http2 bypass
		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
		
		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
		
		$passed_cookies["preferences"] = "date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius";
		
		foreach($passed_cookies as $name => $val){
			
			$header_cookies[] = "$name=$val";
		}
		
		$header_cookies = rtrim(implode("; ", $header_cookies));
		
		if($is_xhr === true){
			
			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
				["User-Agent: " . config::USER_AGENT,
				"Accept: application/json",
				"Accept-Language: en-US,en;q=0.5",
				"Accept-Encoding: gzip",
				"Referer: https://www.startpage.com/",
				"Content-Type: application/json",
				"Content-Length: " . strlen($get),
				"Origin: https://www.startpage.com/",
				"DNT: 1",
				"Connection: keep-alive",
				"Cookie: $header_cookies",
				"Sec-Fetch-Dest: empty",
				"Sec-Fetch-Mode: cors",
				"Sec-Fetch-Site: same-origin",
				"TE: trailers"]
			);
			
		}elseif($post === true){
			
			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
				["User-Agent: " . config::USER_AGENT,
				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
				"Accept-Language: en-US,en;q=0.5",
				"Accept-Encoding: gzip",
				"Referer: https://www.startpage.com/",
				"Content-Type: application/x-www-form-urlencoded",
				"Content-Length: " . strlen($get),
				"DNT: 1",
				"Connection: keep-alive",
				"Cookie: $header_cookies",
				"Upgrade-Insecure-Requests: 1",
				"Sec-Fetch-Dest: document",
				"Sec-Fetch-Mode: navigate",
				"Sec-Fetch-Site: none",
				"Sec-Fetch-User: ?1",
				"Priority: u=0, i",
				"TE: trailers"]
			);
		}else{
			
			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
				["User-Agent: " . config::USER_AGENT,
				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
				"Accept-Language: en-US,en;q=0.5",
				"Accept-Encoding: gzip",
				"DNT: 1",
				"Connection: keep-alive",
				"Cookie: $header_cookies",
				"Sec-Fetch-Dest: document",
				"Sec-Fetch-Mode: navigate",
				"Sec-Fetch-Site: none",
				"Sec-Fetch-User: ?1",
				"Priority: u=0, i",
				"TE: trailers"]
			);
		}
		
		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
		
		$this->backend->assign_proxy($curlproc, $proxy);
		
		$data = curl_exec($curlproc);
		
		if(curl_errno($curlproc)){
			
			throw new Exception(curl_error($curlproc));
		}
		
		curl_close($curlproc);
		
		if($return_cookies){
			
			return [
				"cookies" => $cookies,
				"data" => $data
			];
		}
		
		return $data;
	}
	
	public function web($get){
		
		if($get["npt"]){
			
			$this->anubis_error = "Not ran on pagination";
			[$post, $proxy] = $this->backend->get($get["npt"], "web");
			
			$post = json_decode($post, true);
			$query = $post["query"];
			$cookies = $post["cookies"];
			
			try{
				$html = $this->get(
					$proxy,
					"https://www.startpage.com/sp/search",
					$query,
					true,
					false,
					false,
					$cookies
				);
			}catch(Exception $error){
				
				throw new Exception("Failed to fetch search page");
			}
			
			$get_instant_answer = false;
			
		}else{
			
			$proxy = $this->backend->get_ip();
			
			$params = [
				"query" => $get["s"],
				"cat" => "web",
				"pl" => "opensearch"
			];
			
			if($get["nsfw"] == "no"){
				
				$params["qadf"] = "heavy";
				$get_instant_answer = false;
			}else{
				
				$get_instant_answer = true;
			}
			
			if($get["country"] !== "any"){
				
				$params["qsr"] = $get["country"];
			}
			
			if($get["time"] !== "any"){
				
				$params["with_date"] = $get["time"];
			}
			
			try{
				$html = $this->get(
					$proxy,
					"https://www.startpage.com/sp/search",
					$params,
					false,
					false,
					true,
					[]
				);
			}catch(Exception $error){
				
				throw new Exception("Failed to fetch search page");
			}
			
			//$html = file_get_contents("scraper/startpage.html");
			
			// detect captcha on that page
			$this->detect_captcha($html["data"]);
			
			// solve anubis if need be
			$cookies = $this->solve_anubis($proxy, $html);
			
			if($cookies === false){
				
				$cookies = [];
				$html = $html["data"];
			}else{
				
				try{
					$html = $this->get(
						$proxy,
						"https://www.startpage.com/sp/search",
						$params,
						false,
						false,
						false,
						$cookies
					);
				}catch(Exception $error){
					
					throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
				}
			}
		}
		
		// detect captcha even if we passed anubis challenge
		$this->detect_captcha($html);
		
		//echo htmlspecialchars($html);
		
		if(
			preg_match(
				'/React\.createElement\(UIStartpage\.AppSerpWeb, ?(.+)\),?$/m',
				$html,
				$matches
			) === 0
		){
			
			throw new Exception("Failed to grep JSON object. Anubis error: {$this->anubis_error}");
		}
		
		$json = json_decode($matches[1], true);
		
		if($json === null){
			
			throw new Exception("Failed to decode JSON. Anubis error: {$this->anubis_error}");
		}
		
		//print_r($json);
		
		$out = [
			"status" => "ok",
			"spelling" => [
				"type" => "no_correction",
				"using" => null,
				"correction" => null
			],
			"npt" => null,
			"answer" => [],
			"web" => [],
			"image" => [],
			"video" => [],
			"news" => [],
			"related" => []
		];
		
		// get npt
		$out["npt"] = $this->parse_npt($json, "web", $proxy, $cookies);
		
		foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
			
			if(!isset($category["display_type"])){
				
				continue;
			}
			
			switch($category["display_type"]){
				
				case "web-google":
					foreach($category["results"] as $result){
						
						$sublinks = [];
						
						foreach($result["siteLinks"] as $sublink){
							
							$sublinks[] = [
								"title" => $sublink["title"],
								"description" => null,
								"url" => $sublink["clickUrl"]
							];
						}
						
						$description =
							explode(
								"...",
								$this->titledots(
									html_entity_decode(
										$this->fuckhtml
										->getTextContent(
											$result["description"]
										)
									)
								),
								2
							);
						
						$date = strtotime(trim($description[0]));
						
						if(
							$date === false ||
							count($description) !== 2 ||
							strlen($description[0]) > 14
						){
							
							// no date found
							$description =
								implode(
									" ... ",
									$description
								);
							
							$date = null;
						}else{
							
							// date found
							$description = ltrim($description[1]);
						}
						
						$out["web"][] = [
							"title" =>
								$this->titledots(
									html_entity_decode(
										$this->fuckhtml
										->getTextContent(
											$result["title"]
										)
									)
								),
							"description" => $description,
							"url" => $result["clickUrl"],
							"date" => $date,
							"type" => "web",
							"thumb" => [
								"url" => null,
								"ratio" => null
							],
							"sublink" => $sublinks,
							"table" => []
						];
					}
					break;
				
				case "images-qi-top":
					foreach($category["results"] as $result){
						
						$out["image"][] = [
							"title" =>
								$this->titledots(
									html_entity_decode(
										$this->fuckhtml
										->getTextContent(
											$result["title"]
										)
									)
								),
							"source" => [
								[
									"url" => $result["rawImageUrl"],
									"width" => (int)$result["width"],	
									"height" => (int)$result["height"]
								],
								[
									"url" => $this->unshitimage($result["mdThumbnailUrl"]),
									"width" => (int)$result["mdThumbnailWidth"],
									"height" => (int)$result["mdThumbnailHeight"]
								]
							],
							"url" =>
								$result["altClickUrl"]
						];
					}
					break;
				
				case "spellsuggest-google":
					
					if(isset($category["results"][0]["query"])){
						
						$out["spelling"] =
							[
								"type" => "including",
								"using" => rawurldecode($json["render"]["query"]),
								"correction" => html_entity_decode($category["results"][0]["query"])
							];
					}
					break;
				
				case "dictionary-qi":
					foreach($category["results"] as $result){
						
						$answer = [
							"title" => $result["word"],
							"description" => [],
							"url" => null,
							"thumb" => null,
							"table" => [],
							"sublink" => []
						];
						
						foreach($result["lexical_categories"] as $lexic_type => $definitions){
							
							$answer["description"][] = [
								"type" => "title",
								"value" => $lexic_type
							];
							
							$i = 0;
							
							foreach($definitions as $definition){
								
								$text_definition = trim($definition["definition"]);
								$text_example = trim($definition["example"]);
								$text_synonyms = implode(", ", $definition["synonyms"]);
								
								if($text_definition != ""){
									
									$i++;
									
									$c = count($answer["description"]) - 1;
									if(
										$c !== 0 &&
										$answer["description"][$c]["type"] == "text"
									){
										
										$answer["description"][$c]["value"] .=
											"\n\n" . $i . ". " . $text_definition;
										
									}else{
										
										$answer["description"][] = [
											"type" => "text",
											"value" => $i . ". " . $text_definition
										];
									}
								}
								
								if($text_example != ""){
									
									$answer["description"][] = [
										"type" => "quote",
										"value" => $text_example
									];
								}
								
								if($text_synonyms != ""){
									
									$answer["description"][] = [
										"type" => "text",
										"value" => "Synonyms: " . $text_synonyms
									];
								}
							}
						}
						
						$out["answer"][] = $answer;
					}
					break;
			}
		}
		
		return $out;
	}
	
	public function image($get){
		
		if($get["npt"]){
			
			$this->anubis_error = "Not ran on pagination";
			[$post, $proxy] = $this->backend->get($get["npt"], "images");
			
			$post = json_decode($post, true);
			$query = $post["query"];
			$cookies = $post["cookies"];
			
			try{
				$html = $this->get(
					$proxy,
					"https://www.startpage.com/sp/search",
					$query,
					true,
					false,
					false,
					$cookies
				);
			}catch(Exception $error){
				
				throw new Exception("Failed to fetch search page");
			}
			
		}else{
			
			$search = $get["s"];
			if(strlen($search) === 0){
				
				throw new Exception("Search term is empty!");
			}
			
			$proxy = $this->backend->get_ip();
			
			$params = [
				"query" => $get["s"],
				"cat" => "images",
				"pl" => "opensearch"
			];
			
			if($get["nsfw"] == "no"){
				
				$params["qadf"] = "heavy";
			}
			
			if($get["size"] != "any"){
				
				if(
					$get["size"] == "Small" ||
					$get["size"] == "Medium" ||
					$get["size"] == "Large" ||
					$get["size"] == "Wallpaper"
				){
					
					$params["flimgsize"] = $get["size"];
				}else{
					
					$params["image-size-select"] = "isz:lt,islt:" . $get["size"];
				}
			}
			
			if($get["color"] != "any"){
				
				if($get["color"] == "color"){
					
					$params["flimgcolor"] = "ic:color";
				}elseif($get["color"] == "bnw"){
					
					$params["flimgcolor"] = "ic:gray";
				}else{
					
					$params["flimgcolor"] = "ic:specific,isc:" . $get["color"];
				}
			}
			
			if($get["type"] != "any"){
				
				$params["flimgtype"] = $get["type"];
			}
			
			if($get["license"] != "any"){
				
				$params["flimglicense"] = $get["license"];
			}
			
			try{
				$html = $this->get(
					$proxy,
					"https://www.startpage.com/sp/search",
					$params,
					false,
					false,
					true,
					[]
				);
			}catch(Exception $error){
				
				throw new Exception("Failed to fetch search page");
			}
			
			//$html = file_get_contents("scraper/startpage.html");
			
			// detect captcha on that page
			$this->detect_captcha($html["data"]);
			
			// solve anubis if need be
			$cookies = $this->solve_anubis($proxy, $html);
			
			if($cookies === false){
				
				$cookies = [];
				$html = $html["data"];
			}else{
				
				try{
					$html = $this->get(
						$proxy,
						"https://www.startpage.com/sp/search",
						$params,
						false,
						false,
						false,
						$cookies
					);
				}catch(Exception $error){
					
					throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
				}
			}
		}
		
		$this->detect_captcha($html);
		
		$out = [
			"status" => "ok",
			"npt" => null,
			"image" => []
		];
		
		if(
			preg_match(
				'/React\.createElement\(UIStartpage\.AppSerpImages, ?(.+)\),?$/m',
				$html,
				$matches
			) === 0
		){
			
			throw new Exception("Failed to grep JSON object. Anubis error: {$this->anubis_error}");
		}
		
		$json = json_decode($matches[1], true);
		
		if($json === null){
			
			throw new Exception("Failed to decode JSON object. Anubis error: {$this->anubis_error}");
		}
		
		// get npt
		$out["npt"] = $this->parse_npt($json, "images", $proxy, $cookies);
		
		// get images
		foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
			
			if($category["display_type"] != "images-bing"){
				
				// ignore ads and !! suggestions !! @todo
				continue;
			}
			
			foreach($category["results"] as $image){
				
				$out["image"][] = [
					"title" => $this->titledots($image["title"]),
					"source" => [
						[
							"url" => $this->unshitimage($image["clickUrl"]),
							"width" => (int)$image["width"],
							"height" => (int)$image["height"]
						],
						[
							"url" => $this->unshitimage($image["thumbnailUrl"]),
							"width" => (int)$image["thumbnailWidth"],
							"height" => (int)$image["thumbnailHeight"]
						]
					],
					"url" => $image["altClickUrl"]
				];
			}
		}
		
		return $out;
	}
	
	public function video($get){
		
		if($get["npt"]){
			
			$this->anubis_error = "Not ran on pagination";
			[$post, $proxy] = $this->backend->get($get["npt"], "videos");
			
			$post = json_decode($post, true);
			$query = $post["query"];
			$cookies = $post["cookies"];
			
			try{
				$html = $this->get(
					$proxy,
					"https://www.startpage.com/sp/search",
					$query,
					true,
					false,
					false,
					$cookies
				);
			}catch(Exception $error){
				
				throw new Exception("Failed to fetch search page");
			}
			
		}else{
			
			$search = $get["s"];
			if(strlen($search) === 0){
				
				throw new Exception("Search term is empty!");
			}
			
			$proxy = $this->backend->get_ip();
			
			$params = [
				"query" => $get["s"],
				"cat" => "video",
				"pl" => "opensearch"
			];
			
			if($get["nsfw"] == "no"){
				
				$params["qadf"] = "heavy";
			}
			
			if($get["sort"] != "relevance"){
				
				$params["sort_by"] = $get["sort"];
			}
			
			if($get["duration"] != "any"){
				
				$params["with_duration"] = $get["duration"];
			}
			
			try{
				$html = $this->get(
					$proxy,
					"https://www.startpage.com/sp/search",
					$params,
					false,
					false,
					true,
					[]
				);
			}catch(Exception $error){
				
				throw new Exception("Failed to fetch search page");
			}
			
			//$html = file_get_contents("scraper/startpage.html");
			
			// detect captcha on that page
			$this->detect_captcha($html["data"]);
			
			// solve anubis if need be
			$cookies = $this->solve_anubis($proxy, $html);
			
			if($cookies === false){
				
				$cookies = [];
				$html = $html["data"];
			}else{
				
				try{
					$html = $this->get(
						$proxy,
						"https://www.startpage.com/sp/search",
						$params,
						false,
						false,
						false,
						$cookies
					);
				}catch(Exception $error){
					
					throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
				}
			}
		}
		
		$this->detect_captcha($html);
		
		if(
			preg_match(
				'/React\.createElement\(UIStartpage\.AppSerpVideos, ?(.+)\),?$/m',
				$html,
				$matches
			) === 0
		){
			
			throw new Exception("Failed to get JSON object. Anubis error: {$this->anubis_error}");
		}
		
		$json = json_decode($matches[1], true);
		
		if($json === null){
			
			throw new Exception("Failed to decode JSON object. Anubis error: {$this->anubis_error}");
		}
		
		$out = [
			"status" => "ok",
			"npt" => null,
			"video" => [],
			"author" => [],
			"livestream" => [],
			"playlist" => [],
			"reel" => []
		];
		
		// get npt
		$out["npt"] = $this->parse_npt($json, "video", $proxy, $cookies);
		
		// get results
		foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
			
			if(
				preg_match(
					'/^video-/i',
					$category["display_type"]
				)
			){
				
				foreach($category["results"] as $video){
					
					if(
						isset($video["thumbnailUrl"]) &&
						$video["thumbnailUrl"] !== null
					){
						
						$thumb = [
							"ratio" => "16:9",
							"url" => $this->unshitimage($video["thumbnailUrl"])
						];
					}else{
						
						$thumb = [
							"ratio" => null,
							"url" => null
						];
					}
					
					$out["video"][] = [
						"title" => str_replace(["", ""], "", $video["title"]),
						"description" => $this->limitstrlen($video["description"]),
						"author" => [
							"name" => $video["channelTitle"],
							"url" => null,
							"avatar" => null
						],
						"date" => strtotime($video["publishDate"]),
						"duration" => $this->hms2int($category["display_type"] == "video-youtube" ? $video["duration"] : $video["duration"] / 1000),
						"views" => (int)$video["viewCount"],
						"thumb" => $thumb,
						"url" => $video["clickUrl"]
					];
				}
			}
		}
		
		return $out;
	}
	
	public function news($get){
		
		if($get["npt"]){
			
			$this->anubis_error = "Not ran on pagination";
			[$post, $proxy] = $this->backend->get($get["npt"], "news");
			
			$post = json_decode($post, true);
			$query = $post["query"];
			$cookies = $post["cookies"];
			
			try{
				$html = $this->get(
					$proxy,
					"https://www.startpage.com/sp/search",
					$query,
					true,
					false,
					false,
					$cookies
				);
			}catch(Exception $error){
				
				throw new Exception("Failed to fetch search page");
			}
			
		}else{
			
			$search = $get["s"];
			if(strlen($search) === 0){
				
				throw new Exception("Search term is empty!");
			}
			
			$proxy = $this->backend->get_ip();
			
			$params = [
				"query" => $get["s"],
				"cat" => "news",
				"pl" => "opensearch"
			];
							
			if($get["nsfw"] == "no"){
				
				$params["qadf"] = "heavy";
			}
			
			if($get["time"] != "any"){
				
				$params["with_date"] = $get["time"];
			}
			
			try{
				$html = $this->get(
					$proxy,
					"https://www.startpage.com/sp/search",
					$params,
					false,
					false,
					true,
					[]
				);
			}catch(Exception $error){
				
				throw new Exception("Failed to fetch search page. Anubis error: {$this->anubis_error}");
			}
			
			//$html = file_get_contents("scraper/startpage.html");
			
			// detect captcha on that page
			$this->detect_captcha($html["data"]);
			
			// solve anubis if need be
			$cookies = $this->solve_anubis($proxy, $html);
			
			if($cookies === false){
				
				$cookies = [];
				$html = $html["data"];
			}else{
				
				try{
					$html = $this->get(
						$proxy,
						"https://www.startpage.com/sp/search",
						$params,
						false,
						false,
						false,
						$cookies
					);
				}catch(Exception $error){
					
					throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
				}
			}
		}
		
		$this->detect_captcha($html);
		
		if(
			preg_match(
				'/React\.createElement\(UIStartpage\.AppSerpNews, ?(.+)\),?$/m',
				$html,
				$matches
			) === 0
		){
			
			throw new Exception("Failed to get JSON object. Anubis error: {$this->anubis_error}");
		}
		
		$json = json_decode($matches[1], true);
		
		if($json === null){
			
			throw new Exception("Failed to decode JSON object. Anubis error: {$this->anubis_error}");
		}
		
		$out = [
			"status" => "ok",
			"npt" => null,
			"news" => []
		];
		
		// get npt
		$out["npt"] = $this->parse_npt($json, "news", $proxy, $cookies);
		
		foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
			
			if($category["display_type"] != "news-bing"){
				
				// unsupported category
				continue;
			}
			
			foreach($category["results"] as $news){
				
				if(
					isset($news["thumbnailUrl"]) &&
					$news["thumbnailUrl"] !== null
				){
					
					$thumb = [
						"ratio" => "16:9",
						"url" => $this->unshitimage($news["thumbnailUrl"])
					];
				}else{
					
					$thumb = [
						"ratio" => null,
						"url" => null
					];
				}
				
				$out["news"][] = [
					"title" => $this->titledots($this->remove_penguins($news["title"])),
					"author" => $news["source"],
					"description" => $this->titledots($this->remove_penguins($news["description"])),
					"date" => (int)substr((string)$news["date"], 0, -3),
					"thumb" => $thumb,
					"url" => $news["clickUrl"]
				];
			}
		}
		
		return $out;
	}
	
	private function parse_npt($json, $pagetype, $proxy, $cookies){
		
		foreach($json["render"]["presenter"]["pagination"]["pages"] as $page){
			
			if($page["name"] == "Next"){
				
				return
					$this->backend->store(
						json_encode([
							"query" =>
								http_build_query(
									[
										"lui" => "english",
										"language" => "english",
										"sc" => $json["render"]["search_sc"],
										"t" => "device",
										"cat" => $pagetype,
										"segment" => "startpage.udog", // "original" sometimes
										"abd" => "0",
										"abe" => "0",
										"query" => rawurldecode($json["render"]["query"]),
										"page" => $page["number"],
										"qsr" => "all",
										"qadf" => "none", // @ todo fix (??)
										...(isset($page["token"]) ? ["page_token" => $page["token"]] : [])
									]
								),
							"cookies" => $cookies
						]),
						$pagetype,
						$proxy
					);
				
				break;
			}
		}
		
		return null;
	}
	
	private function unshitimage($url){
		
		$query = parse_url($url, PHP_URL_QUERY);
		parse_str($query, $query);
		
		if(isset($query["piurl"])){
			
			if(strpos($query["piurl"], "gstatic.com/")){
				
				return
					explode(
						"&",
						$query["piurl"],
						2
					)[0];
			}
			
			if(
				strpos($query["piurl"], "bing.net/") ||
				strpos($query["piurl"], "bing.com/")
			){
				
				return
					explode(
						"&",
						$query["piurl"],
						2
					)[0];
			}
			
			return $query["piurl"];
		}
		
		return $url;
	}
	
	private function limitstrlen($text){
		
		return
			explode(
				"\n",
				wordwrap(
					str_replace(
						["\n\r", "\r\n", "\n", "\r"],
						" ",
						$text
					),
					300,
					"\n"
				),
				2
			)[0];
	}
	
	private function titledots($title){
		
		return trim($title, " .\t\n\r\0\x0B…");
	}
	
	private function hms2int($time){
		
		$parts = explode(":", $time, 3);
		$time = 0;
		
		if(count($parts) === 3){
			
			// hours
			$time = $time + ((int)$parts[0] * 3600);
			array_shift($parts);
		}
		
		if(count($parts) === 2){
			
			// minutes
			$time = $time + ((int)$parts[0] * 60);
			array_shift($parts);
		}
		
		// seconds
		$time = $time + (int)$parts[0];
		
		return $time;
	}
	
	private function remove_penguins($text){
		
		return str_replace(
			["", ""],
			"",
			$text
		);
	}
	
	private function detect_captcha($html){
		
		$this->fuckhtml->load($html);
		
		$title =
			$this->fuckhtml
			->getElementsByTagName(
				"title"
			);
		
		if(
			count($title) !== 0 &&
			$title[0]["innerHTML"] == "Redirecting..."
		){
			
			// check if it's a captcha
			$as =
				$this->fuckhtml
				->getElementsByTagName(
					"a"
				);
			
			foreach($as as $a){
				
				if(
					strpos(
						$this->fuckhtml
						->getTextContent(
							$a["innerHTML"]
						),
						"https://www.startpage.com/sp/captcha"
					) !== false
				){
					
					throw new Exception("Startpage returned a captcha");
				}
			}
			
			throw new Exception("Startpage redirected the scraper to an unhandled page");
		}
	}
	
	private function solve_anubis($proxy, $html){
		
		//
		// solve anubis challenge, if theres one
		//
		$res = null;
		$cookies = [];
		
		try{
			$res = $this->anubis->scrape($html["data"]);
			
		}catch(Exception $error){
			
			// do nothang
			$this->anubis_error = $error->getMessage();
		}
		
		if($res !== null){
			
			// https://www.startpage.com/.within.website/x/cmd/anubis/api/pass-challenge
			// ?id=01a00241-da22-7d62-b64f-e7a6a0e8bda8
			// &response=0000460826adc23e0363e97e713e4aa36ca46e1a7cd933184442772f3d6b994c
			// &nonce=3098
			// &redir=https%3A%2F%2Fwww.startpage.com%2Fsp%2Fsearch
			// &elapsedTime=111
			
			$anubis_response =
				$this->get(
					$proxy,
					"https://www.startpage.com/.within.website/x/cmd/anubis/api/pass-challenge",
					[
						"id" => $res["id"],
						"response" => $res["response"],
						"nonce" => $res["nonce"],
						"redir" => "https://www.startpage.com/sp/search",
						"elapsedTime" => $res["elapsedTime"]
					],
					false,
					false,
					true,
					$html["cookies"]
				);
			
			// re-fetch search page AGAIN with the anubis bullshit
			$cookies = array_merge($anubis_response["cookies"], $html["cookies"]);
			
			return $cookies;
		}
		
		return false;
	}
}