4get-mirror/scraper/brave.php

<?php
/*
$brave = new brave();

$handle = fopen("captcha.html", "r");
$html = fread($handle, filesize("captcha.html"));
fclose($handle);

$brave->bypasscaptcha($html, "yes", "ca");*/

class brave{
	
	public function __construct(){
		
		include "lib/fuckhtml.php";
		$this->fuckhtml = new fuckhtml();
		
		include "lib/nextpage.php";
		$this->nextpage = new nextpage("brave");
	}
	
	public function getfilters($page){
		
		switch($page){
			
			case "web":
				return [
					"country" => [
						"display" => "Country",
						"option" => [
							"all" => "All Regions",
							"ar" => "Argentina",
							"au" => "Australia",
							"at" => "Austria",
							"be" => "Belgium",
							"br" => "Brazil",
							"ca" => "Canada",
							"cl" => "Chile",
							"cn" => "China",
							"dk" => "Denmark",
							"fi" => "Finland",
							"fr" => "France",
							"de" => "Germany",
							"hk" => "Hong Kong",
							"in" => "India",
							"id" => "Indonesia",
							"it" => "Italy",
							"jp" => "Japan",
							"kr" => "Korea",
							"my" => "Malaysia",
							"mx" => "Mexico",
							"nl" => "Netherlands",
							"nz" => "New Zealand",
							"no" => "Norway",
							"pl" => "Poland",
							"pt" => "Portugal",
							"ph" => "Philippines",
							"ru" => "Russia",
							"sa" => "Saudi Arabia",
							"za" => "South Africa",
							"es" => "Spain",
							"se" => "Sweden",
							"ch" => "Switzerland",
							"tw" => "Taiwan",
							"tr" => "Turkey",
							"gb" => "United Kingdom",
							"us" => "United States"
						]
					],
					"nsfw" => [
						"display" => "NSFW",
						"option" => [
							"yes" => "Yes",
							"maybe" => "Maybe",
							"no" => "No"
						]
					],
					"newer" => [
						"display" => "Newer than",
						"option" => "_DATE"
					],
					"older" => [
						"display" => "Older than",
						"option" => "_DATE"
					]
				];
				break;
			
			case "news":
				return [
					"country" => [
						"display" => "Country",
						"option" => [
							"all" => "All regions",
							"ar" => "Argentina",
							"au" => "Australia",
							"at" => "Austria",
							"be" => "Belgium",
							"br" => "Brazil",
							"ca" => "Canada",
							"cl" => "Chile",
							"cn" => "China",
							"dk" => "Denmark",
							"fi" => "Finland",
							"fr" => "France",
							"de" => "Germany",
							"hk" => "Hong Kong",
							"in" => "India",
							"id" => "Indonesia",
							"it" => "Italy",
							"jp" => "Japan",
							"kr" => "Korea",
							"my" => "Malaysia",
							"mx" => "Mexico",
							"nl" => "Netherlands",
							"nz" => "New Zealand",
							"no" => "Norway",
							"pl" => "Poland",
							"pt" => "Portugal",
							"ph" => "Philippines",
							"ru" => "Russia",
							"sa" => "Saudi Arabia",
							"za" => "South Africa",
							"es" => "Spain",
							"se" => "Sweden",
							"ch" => "Switzerland",
							"tw" => "Taiwan",
							"tr" => "Turkey",
							"gb" => "United Kingdom",
							"us" => "United States"
						]
					],
					"nsfw" => [
						"display" => "NSFW",
						"option" => [
							"yes" => "Yes",
							"maybe" => "Maybe",
							"no" => "No"
						]
					]
				];
				break;
		}
	}
	
	private function get($url, $get = [], $nsfw, $country/*, $is_post = false, $additional_cookies = null*/){
		
		switch($nsfw){
			
			case "yes": $nsfw = "off"; break;
			case "maybe": $nsfw = "moderate"; break;
			case "no": $nsfw = "strict"; break;
		}
		
		//$cookie = "safesearch={$nsfw}; country={$country}; useLocation=0";
		/*
		if($additional_cookies !== null){
			
			$cookie = $additional_cookies . "; " . $cookie;
		}*/
		
		$headers = [
			"User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/110.0",
			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
			"Accept-Language: en-US,en;q=0.5",
			"Accept-Encoding: gzip",
			"Cookie: safesearch={$nsfw}; country={$country}; useLocation=0; summarizer=0",
			"DNT: 1",
			"Connection: keep-alive",
			"Upgrade-Insecure-Requests: 1",
			"Sec-Fetch-Dest: document",
			"Sec-Fetch-Mode: navigate",
			"Sec-Fetch-Site: none",
			"Sec-Fetch-User: ?1"//,
			//"Content-Type: application/json"
		];
		
		if($country == "any"){
			
			$country = "all";
		}
		
		$curlproc = curl_init();
		
		/*if($is_post){
			
			curl_setopt($curlproc, CURLOPT_POST, true);
			curl_setopt(
				$curlproc,
				CURLOPT_POSTFIELDS,
				json_encode($get)
			);
			
		}else{
			*/
			if($get !== []){
				$get = http_build_query($get);
				$url .= "?" . $get;
			}
		//}
		
		curl_setopt($curlproc, CURLOPT_URL, $url);
		
		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
		curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
		
		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
		
		$data = curl_exec($curlproc);
		
		if(curl_errno($curlproc)){
			
			throw new Exception(curl_error($curlproc));
		}
		
		curl_close($curlproc);
		return $data;
	}
	
	public function web($get){
		
		if($get["npt"]){
			
			// get next page data
			$q = json_decode($this->nextpage->get($get["npt"], "web"), true);
			
			$search = $q["q"];
			$q["spellcheck"] = 0;
			
			$nsfw = $q["nsfw"];
			unset($q["nsfw"]);
			
			$country = $q["country"];
			unset($q["country"]);
			
		}else{
			
			// get _GET data instead
			$search = $get["s"];
			if(strlen($search) === 0){
				
				throw new Exception("Search term is empty!");
			}
			
			if(strlen($search) > 2048){
				
				throw new Exception("Search query is too long!");
			}
			
			$nsfw = $get["nsfw"];
			$country = $get["country"];
			$older = $get["older"];
			$newer = $get["newer"];
			
			$q = [
				"q" => $search
			];
			
			/*
				Pass older/newer filters to brave
			*/
			if($newer !== false){
				
				$newer = date("Y-m-d", $newer);
				
				if($older === false){
					
					$older = date("Y-m-d", time());
				}
			}
			
			if(
				is_string($older) === false &&
				$older !== false
			){
				
				$older = date("Y-m-d", $older);
				
				if($newer === false){
					
					$newer = "1970-01-02";
				}
			}
			
			if($older !== false){
				
				$q["tf"] = "{$newer}to{$older}";
			}
		}
		/*
		$handle = fopen("scraper/brave.html", "r");
		$html = fread($handle, filesize("scraper/brave.html"));
		fclose($handle);
		*/
		try{
			$html =
				$this->get(
					"https://search.brave.com/search",
					$q,
					$nsfw,
					$country
				);
			
		}catch(Exception $error){
			
			throw new Exception("Could not fetch search page");
		}
		
		$out = [
			"status" => "ok",
			"spelling" => [
				"type" => "no_correction",
				"using" => null,
				"correction" => null
			],
			"npt" => null,
			"answer" => [],
			"web" => [],
			"image" => [],
			"video" => [],
			"news" => [],
			"related" => []
		];
		
		// load html
		$this->fuckhtml->load($html);
		
		/*
			Get next page "token"
		*/
		$nextpage =
			$this->fuckhtml
			->getElementsByClassName(
				"btn ml-15",
				"a"
			);
		
		if(count($nextpage) !== 0){
			
			preg_match(
				'/offset=([0-9]+)/',
				$this->fuckhtml->getTextContent($nextpage[0]["attributes"]["href"]),
				$nextpage
			);
				
			$q["offset"] = (int)$nextpage[1];
			$q["nsfw"] = $nsfw;
			$q["country"] = $country;
			
			$out["npt"] =	
				$this->nextpage->store(
					json_encode($q),
					"web"
				);
		}
		
		/*
			Get discussions (and append them to web results)
		*/
		
		// they're loaded using javascript!!
		$discussion =
			$this->fuckhtml
			->getElementById(
				"js-discussions",
				"script"
			);
		
		if(
			$discussion &&
			isset($discussion["attributes"]["data"])
		){
			
			$discussion =
				json_decode(
					$this->fuckhtml
					->getTextContent(
						$discussion["attributes"]["data"]
					),
					true
				);
			
			foreach($discussion["results"] as $result){
				
				$data = [
					"title" => $this->titledots($result["title"]),
					"description" => null,
					"url" => $result["url"],
					"date" => null,
					"type" => "web",
					"thumb" => [
						"url" => null,
						"ratio" => null
					],
					"sublink" => [],
					"table" => []
				];
				
				// description
				$data["description"] =
					$this->limitstrlen(
						$this->limitwhitespace(
							$this->titledots(
								$this->fuckhtml->getTextContent(
									$result["description"]
								)
							)
						)
					);
				
				if($result["age"] != ""){
					$data["date"] = strtotime($result["age"]);
				}
				
				// populate table
				
				if($result["data"]["num_answers"] != ""){
					$data["table"]["Replies"] = (int)$result["data"]["num_answers"];
				}
				
				if($result["data"]["score"] != ""){
					
					$score = explode("|", $result["data"]["score"]);
					
					if(count($score) === 2){
						
						$score = ((int)$score[1]) . " (" . trim($score[0]) . ")";
					}else{
						
						$score = (int)$score[0];
					}
					
					$data["table"]["Votes"] = $score;
				}
				
				if($result["thumbnail"] != ""){
					
					$data["thumb"]["url"] = $result["thumbnail"];
					$data["thumb"]["ratio"] = "16:9";
				}
				
				$out["web"][] = $data;
			}
		}
		
		/*
			Get related searches
		*/
		$faq =
			$this->fuckhtml
			->getElementById("js-faq", "script");
		
		if(
			$faq &&
			isset($faq["attributes"]["data"])
		){
			
			$faq =
				json_decode(
					$this->fuckhtml
					->getTextContent(
						$faq["attributes"]["data"]
					),
					true
				);
			
			foreach($faq["items"] as $related){
				
				$out["related"][] = $related["question"];
			}
		}
		
		/*
			Get spelling autocorrect
		*/
		$altered =
			$this->fuckhtml
			->getElementById("altered-query", "div");
		
		if($altered){
			
			$this->fuckhtml->load($altered);
			
			$altered =
				$this->fuckhtml
				->getElementsByTagName("a");
			
			if(count($altered) === 2){
				
				$out["spelling"] = [
					"type" => "including",
					"using" =>
						$this->fuckhtml
						->getTextContent($altered[0]),
					"correction" =>
						$this->fuckhtml
						->getTextContent($altered[1])
				];
			}
			
			$this->fuckhtml->load($html);
		}
		
		/*
			Get web results
		*/
		$resulthtml =
			$this->fuckhtml
			->getElementById(
				"results",
				"div"
			);
		
		$this->fuckhtml->load($resulthtml);
		$items = 0;
		foreach(
			$this->fuckhtml
			->getElementsByClassName("snippet fdb")
			as $result
		){
			
			$data = [
				"title" => null,
				"description" => null,
				"url" => null,
				"date" => null,
				"type" => "web",
				"thumb" => [
					"url" => null,
					"ratio" => null
				],
				"sublink" => [],
				"table" => []
			];
			
			if(
				isset($result["attributes"]["data-type"]) &&
				$result["attributes"]["data-type"] == "ad"
			){
				
				// is an ad, skip
				continue;
			}
			
			$this->fuckhtml->load($result);
			
			/*
				Get title
			*/
			$title =
				$this->fuckhtml
				->getElementsByClassName(
					"snippet-title",
					"span"
				);
			
			if(count($title) === 0){
				
				// encountered AI summarizer
				// or misspelling indicator @TODO
				continue;
			}
			
			if(isset($title[0]["attributes"]["title"])){
				
				$data["title"] =
					$this->titledots(
						$this->fuckhtml
						->getTextContent(
							$title[0]["attributes"]["title"]
						)
					);
			}else{
				
				$data["title"] =
					$this->titledots(
						$this->fuckhtml
						->getTextContent(
							$title[0]
						)
					);
			}
			
			/*
				Get description
			*/
			$description =
				$this->fuckhtml
				->getElementsByClassName(
					"snippet-description",
					"p"
				);
			
			if(count($description) !== 0){
				$data["description"] =
					$this->titledots(
						$this->fuckhtml
						->getTextContent(
							$description[0]
						)
					);
				
				// also check for thumbnail in here
				$img =
					$this->fuckhtml
					->getElementsByClassName(
						"thumb",
						"img"
					);
				
				if(count($img) !== 0){
					
					$data["thumb"] = [
						"url" => $this->unshiturl($img[0]["attributes"]["src"]),
						"ratio" => "16:9"
					];
				}else{
					
					// might be a video thumbnail wrapper?
					$wrapper =
						$this->fuckhtml
						->getElementsByClassName(
							"video-thumb",
							"a"
						);
					
					if(count($wrapper) !== 0){
						
						// we found a video
						$this->fuckhtml->load($wrapper[0]);
						
						$img =
							$this->fuckhtml
							->getElementsByTagName("img");
						
						$data["thumb"] = [
							"url" => $this->unshiturl($img[0]["attributes"]["src"]),
							"ratio" => "16:9"
						];
						
						// get the video length, if its there
						$duration =
							$this->fuckhtml
							->getElementsByClassName(
								"duration",
								"div"
							);
						
						if(count($duration) !== 0){
							
							$data["table"]["Duration"] = $duration[0]["innerHTML"];
						}
						
						// reset html load
						$this->fuckhtml->load($result);
					}
				}
				
			}else{
				
				// is a steam/shop listing
				$description_alt =
					$this->fuckhtml
					->getElementsByClassName(
						"text-sm",
						"div"
					);
				
				if(count($description_alt) !== 0){
					
					switch($description_alt[0]["attributes"]["class"]){
						
						case "text-sm text-gray":
						case "description text-sm":
							
							$data["description"] =
								$this->titledots(
									$this->fuckhtml
									->getTextContent(
										$description_alt[0]
									)
								);
							break;
					}
					
					// get table sublink
					$sublink =
						$this->fuckhtml
						->getElementsByClassName(
							"r-attr text-sm",
							"div"
						);
					
					if(count($sublink) !== 0){
						
						$this->tablesublink($sublink, $data);
					}
					
					// check for thumb element
					$data["thumb"] = $this->getimagelinkfromstyle("thumb");
				}else{
					
					// ok... finally...
					// maybe its the instant answer thingy
					$answer =
						$this->fuckhtml
						->getElementsByClassName("answer");
					
					if(count($answer) !== 0){
						
						$data["description"] =
							$this->titledots(
								$this->fuckhtml
								->getTextContent($answer[0])
							);
					}
				}
			}
			
			// finally, fix brave's date format sucking balls
			$data["description"] = explode(" - ", $data["description"], 2);
			
			if(count($data["description"]) === 0){
				
				// nothing to do
				$data["description"] = $data["description"][0];
			}else{
				
				// attempt to parse
				$time = strtotime($data["description"][0]);
				
				if($time !== false){
					
					// got response
					$data["date"] = $time;
					
					array_shift($data["description"]);
				}
								
				// merge back
				$data["description"] =
					implode(" - ", $data["description"]);
			}
			
			/*
				Check content type
			*/
			$content_type =
				$this->fuckhtml
				->getElementsByClassName(
					"content-type",
					"span"
				);
			
			if(count($content_type) !== 0){
				
				$data["type"] =
					strtolower($this->fuckhtml->getTextContent($content_type[0]));
			}
			
			/*
				Check subtext table thingy
			*/
			$table_items =
				array_merge(
					$this->fuckhtml
					->getElementsByClassName(
						"item-attributes",
						"div"
					),
					$this->fuckhtml
					->getElementsByClassName(
						"r",
						"div"
					)
				);
			
			/*
				DIV: item-attributes
			*/
			if(count($table_items) !== 0){
				
				foreach($table_items as $table){
					
					$this->fuckhtml->load($table);
					
					$span =
						$this->fuckhtml
						->getElementsByClassName(
							"text-sm",
							"*"
						);
					
					foreach($span as $item){
						
						$item =
							explode(
								":",
								$this->fuckhtml->getTextContent(preg_replace('/\n/', " ", $item["innerHTML"])),
								2
							);
						
						if(count($item) === 2){
							
							$data["table"][trim($item[0])] = trim($this->limitwhitespace($item[1]));
						}
					}
				}
				
				$this->fuckhtml->load($result);
			}
			
			// get video sublinks
			$table_items =
				$this->fuckhtml
				->getElementsByClassName(
					"snippet-description published-time",
					"p"
				);
			
			if(count($table_items) !== 0){
				
				$table_items =
					explode(
						'<span class="mr-15"></span>',
						$table_items[0]["innerHTML"],
						2
					);
				if(count($table_items) === 2){
					
					$item2 = [];
					
					$item2[] = explode(":", $this->fuckhtml->getTextContent($table_items[0]));
					
					if(trim($table_items[1]) != ""){
						$item2[] = explode(":", $this->fuckhtml->getTextContent($table_items[1]));
					}
					
					foreach($item2 as $it){
						
						$data["table"][trim($it[0])] = trim($it[1]);
					}
				}
			}
			
			/*
				Get URL
			*/
			$data["url"] =
				$this->fuckhtml->getTextContent(
					$this->fuckhtml
					->getElementsByTagName("a")
					[0]
					["attributes"]
					["href"]
				);
			
			/*
				Get sublinks
			*/
			$sublinks_elems =
				$this->fuckhtml
				->getElementsByClassName(
					"snippet",
					"div"
				);
			
			$sublinks = [];
			
			foreach($sublinks_elems as $sublink){
				
				$this->fuckhtml->load($sublink);
				
				$a =
					$this->fuckhtml
					->getElementsByTagName("a")[0];
				
				$title =
					$this->fuckhtml
					->getTextContent($a);
				
				$url = $a["attributes"]["href"];
				
				$description =
					$this->titledots(
						$this->fuckhtml
						->getTextContent(
							$this->fuckhtml
							->getElementsByTagName("p")[0]
						)
					);
				
				$sublinks[] = [
					"title" => $title,
					"date" => null,
					"description" => $description,
					"url" => $url
				];
			}
			
			/*
				Get smaller sublinks
			*/
			$sublinks_elems =
				$this->fuckhtml
				->getElementsByClassName(
					"deep-link",
					"a"
				);
			
			foreach($sublinks_elems as $sublink){
				
				$sublinks[] = [
					"title" => $this->fuckhtml->getTextContent($sublink),
					"date" => null,
					"description" => null,
					"url" => $sublink["attributes"]["href"]
				];
			}
			
			// append sublinks to $data !!
			$data["sublink"] = $sublinks;
			
			// append first result to start of $out["web"]
			// other results are after
			if($items === 0){
				
				$out["web"] = [$data, ...$out["web"]];
			}else{
				
				$out["web"][] = $data;
			}
			$items++;
		}
		
		/*
			Get news
		*/
		$this->fuckhtml->load($resulthtml);
		$news_carousel = $this->fuckhtml->getElementById("news-carousel");
		
		$this->fuckhtml->load($news_carousel);
		
		if($news_carousel){
			
			$a =
				$this->fuckhtml
				->getElementsByClassName(
					"card fdb",
					"a"
				);
			
			foreach($a as $news){
				
				$this->fuckhtml->load($news);
				
				$out["news"][] = [
					"title" =>
						$this->titledots(
							$this->fuckhtml
							->getTextContent(
								$this->fuckhtml
								->getElementsByClassName(
									"title",
									"div"
								)[0]
							)
						),
					"description" => null,
					"date" =>
						strtotime(
							$this->fuckhtml
							->getTextContent(
								$this->fuckhtml
								->getElementsByClassName(
									"card-footer__timestamp",
									"span"
								)[0]
							)
						),
					"thumb" => $this->getimagelinkfromstyle("img-bg"),
					"url" => $this->fuckhtml->getTextContent($news["attributes"]["href"])
				];
			}
		}
		
		
		
		/*
			Get videos
		*/
		$this->fuckhtml->load($resulthtml);
		$news_carousel = $this->fuckhtml->getElementById("video-carousel");
		
		$this->fuckhtml->load($news_carousel);
		
		if($news_carousel){
			
			$a =
				$this->fuckhtml
				->getElementsByClassName(
					"card fdb",
					"a"
				);
			
			foreach($a as $video){
				
				$this->fuckhtml->load($video);
				
				$date = null;
				
				$date_o =
					$this->fuckhtml
					->getElementsByClassName(
						"text-gray text-xs",
						"span"
					);
				
				if(count($date_o) !== 0){
					
					$date =
						strtotime(
							$this->fuckhtml
							->getTextContent(
								$date_o[0]
							)
						);
				}
				
				$out["video"][] = [
					"title" =>
						$this->titledots(
							$this->fuckhtml
							->getTextContent(
								$this->fuckhtml
								->getElementsByClassName(
									"title",
									"div"
								)[0]
							)
						),
					"description" => null,
					"date" => $date,
					"duration" => null,
					"views" => null,
					"thumb" => $this->getimagelinkfromstyle("img-bg"),
					"url" => $this->fuckhtml->getTextContent($video["attributes"]["href"])
				];
			}
		}
		
		
		/*
			Get DEFINITION snippet
		*/
		$this->fuckhtml->load($html);
		$infobox = $this->fuckhtml->getElementById("rh-definitions", "div");
		
		if($infobox !== false){
			
			$answer = [
				"title" => null,
				"description" => [],
				"url" => null,
				"thumb" => null,
				"table" => [],
				"sublink" => []
			];
			
			$this->fuckhtml->load($infobox);
			
			$answer["title"] =
				$this->fuckhtml
				->getTextContent(
					$this->fuckhtml
					->getElementsByClassName(
						"header",
						"h5"
					)[0]
				);
			
			$sections =
				$this->fuckhtml
				->getElementsByTagName("section");
			
			$i = -1;
			foreach($sections as $section){
				
				$this->fuckhtml->load($section);
				$items =
					$this->fuckhtml
					->getElementsByTagName("*");
				
				$li = 1;
				$pronounce = false;
				foreach($items as $item){
					
					switch($item["tagName"]){
						
						case "h6":
							
							if(
								isset($item["attributes"]["class"]) &&
								$item["attributes"]["class"] == "h6 pronunciation"
							){
								
								if($pronounce){
									
									break;
								}
								
								$answer["description"][] = [
									"type" => "quote",
									"value" =>
										$this->fuckhtml
										->getTextContent(
											$item
										)
								];
								
								$answer["description"][] =
									[
										"type" => "audio",
										"url" => "https://search.brave.com/api/rhfetch?rhtype=definitions&word={$answer["title"]}&source=ahd-5"
									];
								
								$pronounce = true;
								$i = $i + 2;
								break;
							}
							
							$answer["description"][] = [
								"type" => "title",
								"value" =>
									$this->fuckhtml
									->getTextContent(
										$item
									)
							];
							$i++;
							break;
						
						case "li":
							
							if(
								$i !== -1 &&
								$answer["description"][$i]["type"] == "text"
							){
								
								$answer["description"][$i]["value"] .=
									"\n" . $li . ". " .
									$this->fuckhtml
									->getTextContent(
										$item
									);
								
							}else{
								$answer["description"][] = [
									"type" => "text",
									"value" =>
										$li . ". " .
										$this->fuckhtml
										->getTextContent(
											$item
										)
								];
								$i++;
							}
							$li++;
							break;
						
						case "a":
							$answer["url"] =
								$this->fuckhtml
								->getTextContent(
									$item["attributes"]["href"]
								);
							break;
					}
				}
			}
			
			$out["answer"][] = $answer;
		}
		
		
		/*
			Get instant answer
		*/
		$this->fuckhtml->load($html);
		$infobox = $this->fuckhtml->getElementById("infobox", "div");
		
		if($infobox !== false){
			
			$answer = [
				"title" => null,
				"description" => [],
				"url" => null,
				"thumb" => null,
				"table" => [],
				"sublink" => []
			];
			
			$this->fuckhtml->load($infobox);
			$div = $this->fuckhtml->getElementsByTagName("div");
			
			/*
				Get title + url
			*/
			$title =
				$this->fuckhtml
				->getElementsByClassName("infobox-title", "a");
			
			if(count($title) !== 0){
				
				$answer["title"] =
					$this->fuckhtml
					->getTextContent(
						$title[0]
					);
				
				$answer["url"] =
					$this->fuckhtml
					->getTextContent(
						$title[0]["attributes"]["href"]
					);
			}
			
			/*
				Get thumbnail
			*/
			$thumb = $this->getimagelinkfromstyle("thumb");
			
			if($thumb["url"] !== null){
				
				$answer["thumb"] = $thumb["url"];
			}
			
			/*
				Get table
			*/
			$title =
				$this->fuckhtml
				->getElementsByClassName(
					"infobox-attr-header",
					"div"
				);
			
			$rowhtml = $infobox;
			
			if(count($title) >= 2){
				
				$rowhtml =
					explode(
						$title[1]["outerHTML"],
						$infobox["innerHTML"],
						2
					)[0];
			}
			
			$this->fuckhtml->load($rowhtml);
			
			$rows =
				$this->fuckhtml
				->getElementsByClassName("infobox-attr", "div");
			
			foreach($rows as $row){
				
				if(!isset($row["innerHTML"])){
					
					continue;
				}
				
				$this->fuckhtml->load($row);
				$span =
					$this->fuckhtml
					->getElementsByTagName("span");
				
				if(count($span) === 2){
					
					$answer["table"][
						$this->fuckhtml->getTextContent($span[0])
					] = str_replace("\n", ", ", $this->fuckhtml->getTextContent($span[1], true));
				}
			}
			
			$this->fuckhtml->load($infobox);
			
			/*
				Parse stackoverflow answers
			*/
			$code =
				$this->fuckhtml
				->getElementById("codebox-answer", $div);
			
			if($code){
				
				// this might be standalone text with no paragraphs, check for that
				$author =
					$this->fuckhtml
					->getElementById("author");
				
				$desc_tmp =
					str_replace(
						$author["outerHTML"],
						"",
						$code["innerHTML"]
					);
				
				$this->fuckhtml->load($desc_tmp);
				$code =
					$this->fuckhtml
					->getElementsByTagName("*");
				
				if(count($code) === 0){
					
					$answer["description"] =
					[
						[
							"type" => "text",
							"value" =>
								$this->fuckhtml
								->getTextContent(
									$desc_tmp
								)
						],
						[
							"type" => "quote",
							"value" =>
								$this->fuckhtml
								->getTextContent(
									$author
								)
						]
					];
				}else{
					
					$text = [];
					$i = 0;
					
					foreach($code as $snippet){
						
						switch($snippet["tagName"]){
							
							case "p":
								$this->fuckhtml->load($snippet["innerHTML"]);
								
								$codetags =
									$this->fuckhtml
									->getElementsByTagName("*");
								
								$tmphtml = $snippet["innerHTML"];
								
								foreach($codetags as $tag){
									
									if(!isset($tag["outerHTML"])){
										
										continue;
									}
									
									$tmphtml =
										explode(
											$tag["outerHTML"],
											$tmphtml,
											2
										);
									
									$value = $this->fuckhtml->getTextContent($tmphtml[0], false, false);
									$this->appendtext($value, $text, $i);
									
									$type = null;
									switch($tag["tagName"]){
										
										case "code": $type = "inline_code"; break;
										case "em": $type = "italic"; break;
										case "blockquote": $type = "quote"; break;
										default: $type = "text";
									}
									
									if($type !== null){
										$value = $this->fuckhtml->getTextContent($tag, false, true);
										
										if(trim($value) != ""){
											
											if(
												$i !== 0 &&
												$type == "title"
											){
												
												$text[$i - 1]["value"] = rtrim($text[$i - 1]["value"]);
											}
											
											$text[] = [
												"type" => $type,
												"value" => $value
											];
											$i++;
										}
									}
									
									if(count($tmphtml) === 2){
										
										$tmphtml = $tmphtml[1];
									}else{
										
										break;
									}
								}
								
								if(is_array($tmphtml)){
									
									$tmphtml = $tmphtml[0];
								}
								
								if(strlen($tmphtml) !== 0){
									
									$value = $this->fuckhtml->getTextContent($tmphtml, false, false);
									$this->appendtext($value, $text, $i);
								}
								break;
							
							case "pre":
								
								switch($text[$i - 1]["type"]){
									
									case "text":
									case "italic":
										$text[$i - 1]["value"] = rtrim($text[$i - 1]["value"]);
										break;
								}
								
								$text[] =
									[
										"type" => "code",
										"value" =>
											rtrim(
												$this->fuckhtml
												->getTextContent(
													$snippet,
													true,
													false
												)
											)
									];
								$i++;
								
								break;
							
							case "ol":
								$o = 0;
								
								$this->fuckhtml->load($snippet);
								$li =
									$this->fuckhtml
									->getElementsByTagName("li");
								
								foreach($li as $elem){
									$o++;
									
									$this->appendtext(
										$o . ". " .
										$this->fuckhtml
										->getTextContent(
											$elem
										),
										$text,
										$i
									);
								}
								break;
						}
					}
					
					if(
						$i !== 0 &&
						$text[$i - 1]["type"] == "text"
					){
						
						$text[$i - 1]["value"] = rtrim($text[$i - 1]["value"]);
					}
					
					if($author){
						
						$text[] = [
							"type" => "quote",
							"value" => $this->fuckhtml->getTextContent($author)
						];
					}
					
					$answer["description"] = $text;
				}
			}else{
				
				/*
					Get normal description
				*/
				$description =
					$this->fuckhtml
					->getElementsByClassName(
						"mb-6",
						"div"
					);
				
				if(count($description) !== 0){
					
					$description =
						[
							[
								"type" => "text",
								"value" =>
									$this->titledots(
										preg_replace(
											'/ Wikipedia$/',
											"",
											$this->fuckhtml
											->getTextContent(
												$description[0]
											)
										)
									)
							]
						];
					
					$ratings =
						$this->fuckhtml
						->getElementById("ratings");
					
					if($ratings){
						
						$this->fuckhtml->load($ratings);
						
						$ratings =
							$this->fuckhtml
							->getElementsByClassName(
								"flex-hcenter mb-10",
								"div"
							);
						
						$description[] = [
							"type" => "title",
							"value" => "Ratings"
						];
						
						foreach($ratings as $rating){
							
							$this->fuckhtml->load($rating);
							
							$num =
								$this->fuckhtml
								->getTextContent(
									$this->fuckhtml
									->getElementsByClassName(
										"r-num",
										"div"
									)[0]
								);
							
							$href =
								$this->fuckhtml
								->getElementsByClassName(
									"mr-10",
									"a"
								)[0];
							
							$votes =
								$this->fuckhtml
								->getTextContent(
									$this->fuckhtml
									->getElementsByClassName(
										"text-sm",
										"span"
									)[0]
								);
							
							$c = count($description) - 1;
							
							if(
								$c !== -1 &&
								$description[$c]["type"] == "text"
							){
								
								$description[$c]["value"] .= $num . " ";
							}else{
								
								$description[] = [
									"type" => "text",
									"value" => $num . " "
								];
							}
							
							$description[] = [
								"type" => "link",
								"value" => $this->fuckhtml->getTextContent($href),
								"url" => $this->fuckhtml->getTextContent($href["attributes"]["href"])
							];
							
							$description[] = [
								"type" => "text",
								"value" => " (" . $votes . ")\n"
							];
						}
					}
					
					$answer["description"] = $description;
				}
			}
			
			/*
				Get sublinks
			*/
			$this->fuckhtml->load($infobox);
			
			$profiles =
				$this->fuckhtml
				->getElementById("profiles");
			
			if($profiles){
				$profiles =
					$this->fuckhtml
					->getElementsByClassName(
						"chip",
						"a"
					);
				
				foreach($profiles as $profile){
					
					$name = $this->fuckhtml->getTextContent($profile["attributes"]["title"]);
					
					if(strtolower($name) == "steampowered"){
						
						$name = "Steam";
					}
					
					$answer["sublink"][$name] =
						$this->fuckhtml->getTextContent($profile["attributes"]["href"]);
				}
			}
			
			$actors =
				$this->fuckhtml
				->getElementById("panel-movie-cast");
			
			if($actors){
				
				$this->fuckhtml->load($actors);
				
				$actors =
					$this->fuckhtml
					->getElementsByClassName("card");
				
				$answer["description"][] = [
					"type" => "title",
					"value" => "Cast"
				];
				
				foreach($actors as $actor){
					
					$this->fuckhtml->load($actor);
					
					$answer["description"][] = [
						"type" => "text",
						"value" =>
							$this->fuckhtml
							->getTextContent(
								$this->fuckhtml
								->getElementsByClassName("card-body")
								[0]
							)
					];
					
					$answer["description"][] = [
						"type" => "image",
						"url" => $this->getimagelinkfromstyle("person-thumb")["url"]
					];
				}
			}
			
			$out["answer"][] = $answer;
		}
		
		/*
			Get actor standalone thingy
		*/
		$this->fuckhtml->load($resulthtml);
		$actors =
			$this->fuckhtml
			->getElementById("predicate-entity");
		
		if($actors){
			
			$this->fuckhtml->load($actors);
			
			$cards =
				$this->fuckhtml
				->getElementsByClassName("card");
			
			$url =
				$this->fuckhtml
				->getElementsByClassName(
					"disclaimer",
					"div"
				)[0];
			
			$this->fuckhtml->load($url);
			
			$url =
				$this->fuckhtml
				->getTextContent(
					$this->fuckhtml
					->getElementsByTagName("a")
					[0]
					["attributes"]
					["href"]
				);
			
			$this->fuckhtml->load($actors);
			
			$answer = [
				"title" =>
					$this->fuckhtml
					->getTextContent(
						$this->fuckhtml
						->getElementsByClassName(
							"entity",
							"span"
						)[0]
					) . " (Cast)",
				"description" => [],
				"url" => $url,
				"sublink" => [],
				"thumb" => null,
				"table" => []
			];
			
			foreach($cards as $card){
				
				$this->fuckhtml->load($card);
				
				$answer["description"][] = [
					"type" => "title",
					"value" =>
						$this->fuckhtml
						->getTextContent(
							$this->fuckhtml
							->getElementsByClassName(
								"title"
							)[0]
						)
				];
				
				$answer["description"][] = [
					"type" => "text",
					"value" =>
						$this->fuckhtml
						->getTextContent(
							$this->fuckhtml
							->getElementsByClassName(
								"text-xs desc"
							)[0]
						)
				];
				
				$answer["description"][] = [
					"type" => "image",
					"url" => $this->getimagelinkfromstyle("img-bg")["url"]
				];
			}
			
			$out["answer"][] = $answer;
		}
		
		return $out;
	}
	
	public function news($get){
		
		$search = $get["s"];
		if(strlen($search) === 0){
			
			throw new Exception("Search term is empty!");
		}
		
		$nsfw = $get["nsfw"];
		$country = $get["country"];
		
		if(strlen($search) > 2048){
			
			throw new Exception("Search query is too long!");
		}
		/*
		$handle = fopen("scraper/brave-news.html", "r");
		$html = fread($handle, filesize("scraper/brave-news.html"));
		fclose($handle);*/
		try{
			$html =
				$this->get(
					"https://search.brave.com/news",
					[
						"q" => $search
					],
					$nsfw,
					$country
				);
			
		}catch(Exception $error){
			
			throw new Exception("Could not fetch search page");
		}
		
		$out = [
			"status" => "ok",
			"npt" => null,
			"news" => []
		];
		
		// load html
		$this->fuckhtml->load($html);
		
		$news =
			$this->fuckhtml
			->getElementsByClassName(
				"snippet inline gap-standard",
				"div"
			);
		
		foreach($news as $article){
			
			$data = [
				"title" => null,
				"author" => null,
				"description" => null,
				"date" => null,
				"thumb" =>
					[
						"url" => null,
						"ratio" => null
					],
				"url" => null
			];
			
			$this->fuckhtml->load($article);
			$elems =
				$this->fuckhtml
				->getElementsByTagName("*");
			
			// get title
			$data["title"] =
				$this->fuckhtml
				->getTextContent(
					$this->fuckhtml
					->getElementsByClassName(
						"snippet-title",
						$elems
					)
					[0]
					["innerHTML"]
				);
			
			// get description
			$data["description"] =
				$this->titledots(
					$this->fuckhtml
					->getTextContent(
						$this->fuckhtml
						->getElementsByClassName(
							"snippet-description",
							$elems
						)
						[0]
						["innerHTML"]
					)
				);
			
			// get date
			$date =
				explode(
					"•",
					$this->fuckhtml
					->getTextContent(
						$this->fuckhtml
						->getElementsByClassName(
							"snippet-url",
							$elems
						)[0]
					)
				);
			
			if(
				count($date) !== 1 &&
				trim($date[1]) != ""
			){
				
				$data["date"] =
					strtotime(
						$date[1]
					);
			}
			
			// get URL
			$data["url"] =
				$this->fuckhtml->getTextContent(
					$this->unshiturl(
						$this->fuckhtml
						->getElementsByClassName(
							"result-header",
							$elems
						)
						[0]
						["attributes"]
						["href"]
					)
				);
			
			// get thumbnail
			$thumb =
				$this->fuckhtml
				->getElementsByTagName(
					"img"
				);
			
			if(
				count($thumb) === 2 &&
				trim(
					$thumb[1]
					["attributes"]
					["src"]
				) != ""
			){
				
				$data["thumb"] = [
					"url" =>
						$this->fuckhtml->getTextContent(
							$this->unshiturl(
								$thumb[1]
								["attributes"]
								["src"]
							)
						),
					"ratio" => "16:9"
				];
			}
			
			$out["news"][] = $data;
		}
		
		return $out;
	}
	
	/*
	public function bypasscaptcha($html, $nsfw, $country){
		
		// @TODO figure out why I still cant go trough
		// the captcha wall even after breaking it
		
		try{
			$html =
				$this->get(
					"https://search.brave.com/goggles",
					[
						"q" => "site:dailymotion.com my bloody valentine"
					],
					$nsfw,
					$country
				);
			
		}catch(Exception $error){
			
			throw new Exception("Could not fetch html");
		}
		
		// Bypass brave search captcha
		// this captcha only appears on the goggles page
		preg_match(
			'/this\.img\.src = &#34;(.*)&#34;/',
			$html,
			$image
		);
		
		$image =
			base64_decode(
				explode(
					"data:image/png;base64,",
					$image[1]
				)[1]
			);
		
		$im = new Imagick();
		$im->readImageBlob($image);
		
		$im->blurImage(20, 20);
		$im->posterizeImage(2, imagick::IMGTYPE_COLORSEPARATION);
		
		// if we encounter a white line thats longer than 45px
		// we found the circle position
		$iterator = $im->getPixelRegionIterator(0, 77, 310, 1);
		
		$found = null;
		foreach(
			$iterator as $row
		){
			
			$whitecount = 0;
			$count = 0;
			
			foreach($row as $pixel){
				
				if($pixel->getColor()["r"] === 255){
					
					$whitecount++;
					$pixel->setColor("rgba(255,0,0,0)");
					
					if($whitecount === 45){
						
						$found = $count - 45;
						break 2;
					}
				}else{
					
					$whitecount = 0;
				}
				
				$count++;
				$iterator->syncIterator();
			}
		}
		
		$found = $found + 10;
		
		//header("Content-Type: image/png");
		//echo $im;
		//die();
		
		if($found === null){
			
			throw new Exception("Could not bypass captcha");
		}
		
		preg_match(
			'/data="{&#34;captcha_id&#34;:&#34;([0-9A-z-]+)&#34;}"/',
			$html,
			$key
		);
		
		$key = $key[1];
		// we bypassed captcha, send POST data
		$order =
			$this->get(
				"https://search.brave.com/api/captcha?brave=0&captcha_id={$key}",
				[
					"solution" => (string)$found
				],
				$nsfw,
				$country,
				true
			);
		
		$order = json_decode($order, true)["orderId"];
		
		$orderpayload =
			$this->get(
				"https://search.brave.com/api/rewards/v1/orders/{$order}",
				[],
				$nsfw,
				$country
			);
		
		$orderpayload = json_decode($orderpayload, true);
		
		$creds =
			$this->get(
				"https://search.brave.com/api/rewards/v1/orders/{$order}/credentials",
				[
					"itemId" => $orderpayload["items"][0]["id"],
					"blindedCreds" => [
						"fuYAVcB/m7BU66vf3wkNGxJCSaRhshB9o+8km3F1h2c=",
						"uswvcWJuPK/1qFlVdzBP3eQd0+V1EQgfAtnEoMIK+Uk=",
						"fJWKGLBxl3Gyn4n9FjTLq1PjupfABT7Ni8MeB+iGzUs=",
						"Aq9enJ/VZP9GxQIza3n65ZK7xQhY4VwDxv53BCb/Txg=",
						"FMJA9eSLHq71K+Pcwgm4gIQOmdR/6KMy5cMgXhpd5Ro=",
						"2NVhIAbvI317SP9/xXbVe/U57eWgvHyqVbHL/5+Gdmw=",
						"6mpjsjSCmYEzK2xlbL8DI2P4LuhWUOxjTLvsTAL9l24=",
						"kAn4wuHvIlKWhfuFfPTSfD4tZ5le9t7/61YbdEc/L3k=",
						"BjjUyG16aTfd1c0h4oBzgQQOekrH1f+a5CmcXqMPTR4=",
						"SBNgpCt4/V44yaQTfh+D027Yv1GJFHkjUEpPw6rAwRI=",
						"XDENAtdQ7PyYx+Qx1wQGQtDWgg8WpIMgWGmd4RDOVWE=",
						"tF7rB4sqamsiUk3K7fojdQSI0Q6iip72yKyhnvg/bC0=",
						"VsAqflirAd/u4VsLdfRS2UvnH24ZNkFh6YN3DctLjzQ=",
						"MntLbXkoI0LdcisCbNazmooiHXJyX91L1KERDAu1JRU=",
						"TH6Zs8JBvFDbTDWgKbfGE4M5/cSwCtHD8ms5Y/U8zHQ=",
						"jsZg0Z+qDPHymrbhdnesodhLNJ26QdunyMko1aVe4So=",
						"rpKsyj6/vdnuMgLI2BApeijtGq9g5USRDL0w6X2bnlQ=",
						"vCzliGT8A9vcLXj2sFf2kavOuYw69d70NpfgA22B4lI=",
						"7OWoxSCtYXWcaBSifF7AXNBif/sjcuO0IelzXG/3PFk=",
						"iiXtByNlT6nDMN9De5B58Jl8J0p6LCjnZ9aS3w2FEQU=",
						"zDhd7gsJ4h4JkDeGK0Y0mfFd8IBdkLhMOANzwO+4Dig=",
						"qANZ+AikwFReEA61JF009d/c3IHM/aSfIYwljckhJWE=",
						"nNC30pDLxtXvUr+WDwfDSrAInNBpfSZkPsV2JlpheWI=",
						"kGXE1pkt25P71kdJzmKIg4+yMR1VA5wNmbpBb/FhJQ8=",
						"aLqPsY1Qiz2UCa2Jx3YNNt8r4JINMphks/43EiyZfXU=",
						"bHGYZoQARZEM5LdFF6B74PkRqNd9EKxzuTvGYxjq+hk=",
						"JOsYQjfE/9Y1u29hR+GvEkNyxUI8blgLhX1iJI/aGRQ=",
						"yKjHjH5j600TJD/3WPsA1N3OmItDLifdjlysq4H6NV0=",
						"9lTnUbsPp7BJ7XVN5/T4yGfzD9DJdqWB7xk72s19MAA=",
						"5KHG8iY45em7zDhO/HlI0ydcZ0Ubn+XSyjifMmy7qXM="
					]
				],
				$nsfw,
				$country,
				true
			);
		
		var_dump($creds);
		
		sleep(2);
		$test =
			$this->get(
				"https://search.brave.com/api/rewards/v1/orders/{$order}/credentials",
				[],
				$nsfw,
				$country
			);
		
		var_dump($test);
		
		$html =
			$this->get(
				"https://search.brave.com/goggles",
				[
					"q" => "site:dailymotion.com my bloody valentine"
				],
				$nsfw,
				$country,
				false,
				"__Secure-sku#brave-search-captcha=eyJ0eXBlIjoic2luZ2xlLXVzZSIsInZlcnNpb24iOjEsInNrdSI6ImJyYXZlLXNlYXJjaC1jYXB0Y2hhIiwicHJlc2VudGF0aW9uIjoiZXlKcGMzTjFaWElpT2lKaWNtRjJaUzVqYjIwL2MydDFQV0p5WVhabExYTmxZWEpqYUMxallYQjBZMmhoSWl3aWMybG5ibUYwZFhKbElqb2lNRzl0VDBneWQxZ3dTazkzU0VFMVJ6QTJaR1V5WjFOQ1dDdGhSM3B2Y2xsTVQwVTJZVVJtTUc5a1IweG1Wa3RhZEd0cU4xbHdia3BPT0VOVGNGbE5lVWR2YmpGRlNTOUhhMlZYU1RWNGQxTjJPWGxJTTNjOVBTSXNJblFpT2lKWlJWWldaVzR5TTJwQ01tSnZkakJ2U1hGNGJtSndUMGxEUW5Kd1drRjBRbWQxVnpoRlNURTNVREY2UVRaQlpUTXJSVGRFYm5NeVFqUmhka0pGYTFWM2FGY3JWRVZJVjNWcE9TdFllRU1yYlVSTVkyMTBRVDA5SW4wPSJ9"
			);
		
		var_dump($html);
	}*/
	
	private function appendtext($payload, &$text, &$index){
		
		if(trim($payload) == ""){
			
			return;
		}
		
		if(
			$index !== 0 &&
			$text[$index - 1]["type"] == "text"
		){
			
			$text[$index - 1]["value"] .= "\n\n" . preg_replace('/  $/', " ", $payload);
		}else{
			
			$text[] = [
				"type" => "text",
				"value" => preg_replace('/  $/', " ", $payload)
			];
			$index++;
		}
	}
	
	private function tablesublink($html_collection, &$data){
		
		foreach($html_collection as $html){
			
			$html["innerHTML"] = preg_replace(
				'/<style>[\S\s]*<\/style>/i',
				"",
				$html["innerHTML"]
			);
			
			$html =
				explode(
					":",
					$this->fuckhtml->getTextContent($html),
					2
				);
			
			if(count($html) === 1){
				
				$html = ["Rating", $html[0]];
			}
			
			$data["table"][trim($html[0])] = trim($html[1]);
		}
	}
	
	private function getimagelinkfromstyle($thumb){
		
		$thumb =
			$this->fuckhtml
			->getElementsByClassName(
				$thumb,
				"div"
			);
		
		if(count($thumb) === 0){
			
			return [
				"url" => null,
				"ratio" => null
			];
		}
		
		$thumb = $thumb[0]["attributes"]["style"];
		
		preg_match(
			'/background-image: ?url\((\'[^\']+\'|"[^"]+"|[^\)]+)\)/',
			$thumb,
			$thumb
		);
		
		$url = $this->fuckhtml->getTextContent($this->unshiturl(trim($thumb[1], '"\' ')));
		
		if(parse_url($url, PHP_URL_HOST) == "cdn.search.brave.com"){
			
			return [
				"url" => null,
				"ratio" => null
			];
		}
		
		return [
			"url" => $url,
			"ratio" => "16:9"
		];
	}
	
	private function limitstrlen($text){
		
		return explode("\n", wordwrap($text, 300, "\n"))[0];
	}
	
	private function limitwhitespace($text){
		
		return
			preg_replace(
				'/[\s]+/',
				" ",
				$text
			);
	}
	
	private function titledots($title){
		
		$substr = substr($title, -3);
		
		if(
			$substr == "..." ||
			$substr == "…"
		){
						
			return trim(substr($title, 0, -3));
		}
		
		return trim($title);
	}
	
	private function unshiturl($url){
		
		// https://imgs.search.brave.com/XFnbR8Sl7ge82MBDEH7ju0UHImRovMVmQ2qnDvgNTuA/rs:fit:844:225:1/g:ce/aHR0cHM6Ly90c2U0/Lm1tLmJpbmcubmV0/L3RoP2lkPU9JUC54/UWotQXU5N2ozVndT/RDJnNG9BNVhnSGFF/SyZwaWQ9QXBp.jpeg
		
		$tmp = explode("aHR0", $url);
		
		if(count($tmp) !== 2){
			
			// nothing to do
			return $url;
		}
		
		return
			base64_decode(
				"aHR0" .
				str_replace(["/", "_"], ["", "/"],
					explode(
						".",
						$tmp[1]
					)[0]
				)
			);
	}
}