4get-mirror/scraper/yep.php

<?php

class yep{
	
	public function __construct(){
		
		include "lib/backend.php";
		$this->backend = new backend("yep");
		
		include "lib/fuckhtml.php";
		$this->fuckhtml = new fuckhtml();
	}
	
	public function getfilters($page){
		
		return [
			"lang" => [
				"display" => "Language",
				"option" => [
					"any" => "Any language",
					"aa" => "Afar",
					"ab" => "Abkhazian",
					"ae" => "Avestan",
					"af" => "Afrikaans",
					"ak" => "Akan",
					"am" => "Amharic",
					"an" => "Aragonese",
					"ar" => "Arabic",
					"as" => "Assamese",
					"av" => "Avaric",
					"ay" => "Aymara",
					"az" => "Azerbaijani",
					"ba" => "Bashkir",
					"be" => "Belarusian",
					"bg" => "Bulgarian",
					"bh" => "Bihari",
					"bi" => "Bislama",
					"bm" => "Bambara",
					"bn" => "Bengali",
					"bo" => "Tibetan",
					"br" => "Breton",
					"bs" => "Bosnian",
					"ca" => "Catalan",
					"ce" => "Chechen",
					"ch" => "Chamorro",
					"co" => "Corsican",
					"cr" => "Cree",
					"cs" => "Czech",
					"cu" => "Church Slavic",
					"cv" => "Chuvash",
					"cy" => "Welsh",
					"da" => "Danish",
					"de" => "German",
					"dv" => "Divehi",
					"dz" => "Dzongkha",
					"ee" => "Ewe",
					"el" => "Greek",
					"en" => "English",
					"eo" => "Esperanto",
					"es" => "Spanish",
					"et" => "Estonian",
					"eu" => "Basque",
					"fa" => "Persian",
					"ff" => "Fulah",
					"fi" => "Finnish",
					"fj" => "Fijian",
					"fo" => "Faroese",
					"fr" => "French",
					"fy" => "Western Frisian",
					"ga" => "Irish",
					"gd" => "Scottish Gaelic",
					"gl" => "Galician",
					"gn" => "Guarani",
					"gu" => "Gujarati",
					"gv" => "Manx",
					"ha" => "Hausa",
					"he" => "Hebrew",
					"hi" => "Hindi",
					"ho" => "Hiri Motu",
					"hr" => "Croatian",
					"ht" => "Haitian",
					"hu" => "Hungarian",
					"hy" => "Armenian",
					"hz" => "Herero",
					"ia" => "Interlingua",
					"id" => "Indonesian",
					"ie" => "Interlingue",
					"ig" => "Igbo",
					"ii" => "Sichuan Yi",
					"ik" => "Inupiaq",
					"io" => "Ido",
					"is" => "Icelandic",
					"it" => "Italian",
					"iu" => "Inuktitut",
					"ja" => "Japanese",
					"jv" => "Javanese",
					"ka" => "Georgian",
					"kg" => "Kongo",
					"ki" => "Kikuyu",
					"kj" => "Kuanyama",
					"kk" => "Kazakh",
					"kl" => "Kalaallisut",
					"km" => "Central Khmer",
					"kn" => "Kannada",
					"ko" => "Korean",
					"kr" => "Kanuri",
					"ks" => "Kashmiri",
					"ku" => "Kurdish",
					"kv" => "Komi",
					"kw" => "Cornish",
					"ky" => "Kyrgyz",
					"la" => "Latin",
					"lb" => "Luxembourgish",
					"lg" => "Ganda",
					"li" => "Limburgish",
					"ln" => "Lingala",
					"lo" => "Lao",
					"lt" => "Lithuanian",
					"lu" => "Luba-Katanga",
					"lv" => "Latvian",
					"mg" => "Malagasy",
					"mh" => "Marshallese",
					"mi" => "Maori",
					"mk" => "Macedonian",
					"ml" => "Malayalam",
					"mn" => "Mongolian",
					"mr" => "Marathi",
					"ms" => "Malay",
					"mt" => "Maltese",
					"my" => "Burmese",
					"na" => "Nauru",
					"nb" => "Norwegian Bokmål",
					"nd" => "North Ndebele",
					"ne" => "Nepali",
					"ng" => "Ndonga",
					"nl" => "Dutch",
					"nn" => "Norwegian Nynorsk",
					"no" => "Norwegian",
					"nr" => "South Ndebele",
					"nv" => "Navajo",
					"ny" => "Chichewa",
					"oc" => "Occitan",
					"oj" => "Ojibwa",
					"om" => "Oromo",
					"or" => "Oriya",
					"os" => "Ossetian",
					"pa" => "Punjabi",
					"pi" => "Pali",
					"pl" => "Polish",
					"ps" => "Pashto",
					"pt" => "Portuguese",
					"qu" => "Quechua",
					"rm" => "Romansh",
					"rn" => "Rundi",
					"ro" => "Romanian",
					"ru" => "Russian",
					"rw" => "Kinyarwanda",
					"sa" => "Sanskrit",
					"sc" => "Sardinian",
					"sd" => "Sindhi",
					"se" => "Northern Sami",
					"sg" => "Sango",
					"si" => "Sinhala",
					"sk" => "Slovak",
					"sl" => "Slovenian",
					"sm" => "Samoan",
					"sn" => "Shona",
					"so" => "Somali",
					"sq" => "Albanian",
					"sr" => "Serbian",
					"ss" => "Swati",
					"st" => "Southern Sotho",
					"su" => "Sundanese",
					"sv" => "Swedish",
					"sw" => "Swahili",
					"ta" => "Tamil",
					"te" => "Telugu",
					"tg" => "Tajik",
					"th" => "Thai",
					"ti" => "Tigrinya",
					"tk" => "Turkmen",
					"tl" => "Tagalog",
					"tn" => "Tswana",
					"to" => "Tonga",
					"tr" => "Turkish",
					"ts" => "Tsonga",
					"tt" => "Tatar",
					"tw" => "Twi",
					"ty" => "Tahitian",
					"ug" => "Uyghur",
					"uk" => "Ukrainian",
					"ur" => "Urdu",
					"uz" => "Uzbek",
					"ve" => "Venda",
					"vi" => "Vietnamese",
					"vo" => "Volapük",
					"wa" => "Walloon",
					"wo" => "Wolof",
					"xh" => "Xhosa",
					"yi" => "Yiddish",
					"yo" => "Yoruba",
					"za" => "Zhuang",
					"zh" => "Chinese",
					"zh-cn" => "Chinese (Simplified)",
					"zh-tw" => "Chinese (Traditional)",
					"zu" => "Zulu"
				]
			]
		];
	}
	
	private function get($proxy, $url, $get = [], $use_api = false, $post_data = null, $bearer = null){
		
		$curlproc = curl_init();
		
		if($get !== []){
			$get = http_build_query($get);
			$url .= "?" . $get;
		}
		
		curl_setopt($curlproc, CURLOPT_URL, $url);
		
		// use http1 lol
		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
		
		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
		
		if($use_api){
			
			$post_data = json_encode($post_data);
			
			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
				["Content-Type: application/json",
				"Authorization: Bearer $bearer",
				"Content-Length: " . strlen($post_data)]
			);
			
			curl_setopt($curlproc, CURLOPT_POST, true);
			curl_setopt($curlproc, CURLOPT_POSTFIELDS, $post_data);
		}else{
			
			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
				["User-Agent: " . config::USER_AGENT,
				"Accept: */*",
				"Accept-Language: en-US,en;q=0.9",
				"Accept-Encoding: gzip, deflate, br, zstd",
				"Referer: https://yep.com/",
				"Origin: https://yep.com",
				"DNT: 1",
				"Connection: keep-alive",
				"Sec-Fetch-Dest: empty",
				"Sec-Fetch-Mode: cors",
				"Sec-Fetch-Site: same-site",
				"Priority: u=4",
				"Pragma: no-cache",
				"Cache-Control: no-cache",
				"TE: trailers"]
			);
		}
		
		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
		
		$this->backend->assign_proxy($curlproc, $proxy);
		
		$data = curl_exec($curlproc);
		
		if(curl_errno($curlproc)){
			
			throw new Exception(curl_error($curlproc));
		}
		
		curl_close($curlproc);
		return $data;
	}
	
	
	
	public function web($get){
		
		if(config::YEP_USE_API){
			
			return $this->web_api($get);
		}
		
		$search = $get["s"];
		if(strlen($search) === 0){
			
			throw new Exception("Search term is empty!");
		}
		
		$out = [
			"status" => "ok",
			"spelling" => [
				"type" => "no_correction",
				"using" => null,
				"correction" => null
			],
			"npt" => null,
			"answer" => [],
			"web" => [],
			"image" => [],
			"video" => [],
			"news" => [],
			"related" => []
		];
		
		// parse filters
		$filters = [
			"q" => $search,
			"limit" => 10 // no fun allowed
		];
		
		if($get["lang"] != "any"){
			
			$filters["language"] = $get["lang"];
			$filters["lang"] = $get["lang"];
		}
		
		try{
			
			// https://yep.com/web/v1/search?q=asmr&limit=10
			$json =
				$this->get(
					$this->backend->get_ip(),
					"https://yep.com/web/v1/search",
					$filters
				);
			
		}catch(Exception $error){
			
			throw new Exception("Failed to fetch JSON");
		}
		
		$this->detect_cf($json);
		
		$json = json_decode($json, true);
		//$json = json_decode(file_get_contents("scraper/yep.json"), true);
		if($json === null){
			
			throw new Exception("Failed to decode JSON");
		}
		
		if(isset($json["code"])){
			
			if(isset($json["reason"])){
				
				throw new Exception("Yep returned an API error: [{$json["code"]}] {$json["reason"]}");
			}
			
			throw new Exception("Yep returned an unknown error");
		}
		
		if(isset($json["results"])){
			foreach($json["results"] as $item){
				
				$out["web"][] = [
					"title" => $item["title"],
					"description" => str_replace("\n", " ", $item["description"]),
					"url" => $item["url"],
					"date" => null,
					"type" => "web",
					"thumb" => [
						"url" => null,
						"ratio" => null
					],
					"sublink" => [],
					"table" => []
				];
			}
		}
		
		return $out;
	}
	
	
	private function web_api($get){
		
		$search = $get["s"];
		if(strlen($search) === 0){
			
			throw new Exception("Search term is empty!");
		}
		
		$out = [
			"status" => "ok",
			"spelling" => [
				"type" => "no_correction",
				"using" => null,
				"correction" => null
			],
			"npt" => null,
			"answer" => [],
			"web" => [],
			"image" => [],
			"video" => [],
			"news" => [],
			"related" => []
		];
		
		// parse filters
		$filters = [
			"query" => $search,
			"limit" => 99 // for some reason they dont show anything sometimes at 100 results
		];
		
		if($get["lang"] != "any"){ $filters["language"] = [ $get["lang"] ]; }
		
		// add api key
		$key_data = $this->backend->get_key();
		
		try{
			
			$json =
				$this->get(
					$this->backend->get_ip($key_data["increment"]),
					"https://platform.yep.com/api/search",
					[],
					true,
					$filters,
					$key_data["key"]
				);
			
		}catch(Exception $error){
			
			throw new Exception("Failed to fetch JSON");
		}
		
		// should never happen
		//$this->detect_cf($json);
		
		$json = json_decode($json, true);
		//$json = json_decode(file_get_contents("scraper/yep.json"), true);
		
		if($json === null){
			
			throw new Exception("Failed to decode JSON");
		}
		
		if(isset($json["error"])){
			
			throw new Exception("Yep API returned an error: " . $json["error"]);
		}
		
		if(isset($json["errors"])){
			
			throw new Exception("Yep API returned the following errors: {$json["message"]}");
		}
		
		if(
			isset($json["success"]) &&
			$json["success"] !== true
		){
			
			throw new Exception("Yep API returned a false-y success value");
		}
		
		if(!isset($json["results"])){
			
			throw new Exception("Yep API did not return a results object");
		}
		
		foreach($json["results"] as $item){
			
			if(
				$item["url"] === null ||
				$item["url"] == ""
			){
				
				// sometimes API fucks up
				continue;
			}
			
			$out["web"][] = [
				"title" => $item["title"],
				"description" => $item["description"],
				"url" => $item["url"],
				"date" => null,
				"type" => "web",
				"thumb" => [
					"url" => null,
					"ratio" => null
				],
				"sublink" => [],
				"table" => []
			];
		}
		
		return $out;
	}
	
	
	private function detect_cf($payload){
		
		// detect cloudflare page
		$this->fuckhtml->load($payload);
		
		if(
			count(
				$this->fuckhtml
				->getElementsByClassName(
					"cf-wrapper",
					"div"
				)
			) !== 0
		){
			
			throw new Exception("Blocked by Cloudflare. Please follow curl-impersonate installation instructions");
		}
	}
	
	
	private function titledots($title){
		
		$substr = substr($title, -4);
		
		if(
			strpos($substr, "...") !== false ||
			strpos($substr, "…") !== false
		){
						
			return trim(substr($title, 0, -4));
		}
		
		return trim($title);
	}
	
	private function unshiturl($url){
		
		$newurl = parse_url($url, PHP_URL_QUERY);
		parse_str($newurl, $newurl);
		
		if(isset($newurl["url"])){
			
			return $newurl["url"];
		}
		
		return $url;
	}
}