lolcat 2 недель назад
Родитель
Сommit
25d4f991c2
5 измененных файлов с 180 добавлено и 139 удалено
  1. 2 0
      api/v1/ac.php
  2. 165 137
      scraper/brave.php
  3. 1 1
      scraper/google.php
  4. 2 0
      scraper/google_api.php
  5. 10 1
      scraper/yt.php

+ 2 - 0
api/v1/ac.php

@@ -163,6 +163,8 @@ class autocomplete{
 				// if it respects the openSearch protocol
 				$json = json_decode($this->get($this->scrapers[$scraper], $_GET["s"]), true);
 				
+				if(!isset($json[1])){ $json[1] = []; }
+				
 				echo json_encode(
 					[
 						$_GET["s"],

+ 165 - 137
scraper/brave.php

@@ -460,7 +460,7 @@ class brave{
 				){
 					
 					$thumb = [
-						"ratio" => $result["thumbnail"]["logo"] == "false" ? "16:9" : "1:1",
+						"ratio" => (isset($result["thumbnail"]["logo"]) && $result["thumbnail"]["logo"] == "true") ? "1:1" : "16:9",
 						"url" => $result["thumbnail"]["original"]
 					];
 				}else{
@@ -994,6 +994,15 @@ class brave{
 				
 				foreach($data["videos"]["results"] as $video){
 					
+					if(
+						$video["title"] == "" &&
+						$video["description"] == ""
+					){
+						
+						// ignore result
+						continue;
+					}
+					
 					// get thumbnails
 					$thumbs = [];
 					
@@ -1111,141 +1120,6 @@ class brave{
 		return $out;
 	}
 	
-	public function news($get){
-		
-		if($get["npt"]){
-			
-			[$req, $proxy] = $this->backend->get($get["npt"], "news");
-			
-			$req = json_decode($req, true);
-			
-			$search = $req["q"];
-			$country = $req["country"];
-			$nsfw = $req["nsfw"];
-			$offset = $req["offset"];
-			$spellcheck = $req["spellcheck"];
-			
-			try{
-				$html =
-					$this->get(
-						$proxy,
-						"https://search.brave.com/news",
-						[
-							"q" => $search,
-							"offset" => $offset,
-							"spellcheck" => $spellcheck
-						],
-						$nsfw,
-						$country
-					);
-				
-			}catch(Exception $error){
-				
-				throw new Exception("Could not fetch search page");
-			}
-			
-		}else{
-			$search = $get["s"];
-			if(strlen($search) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			if(strlen($search) > 2048){
-				
-				throw new Exception("Search term is too long!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-			$nsfw = $get["nsfw"];
-			$country = $get["country"];
-			$spellcheck = $get["spellcheck"] == "yes" ? "1" : "0";
-			
-			/*
-			$handle = fopen("scraper/brave-news.html", "r");
-			$html = fread($handle, filesize("scraper/brave-news.html"));
-			fclose($handle);*/
-			try{
-				$html =
-					$this->get(
-						$proxy,
-						"https://search.brave.com/news",
-						[
-							"q" => $search,
-							"spellcheck" => $spellcheck
-						],
-						$nsfw,
-						$country
-					);
-				
-			}catch(Exception $error){
-				
-				throw new Exception("Could not fetch search page");
-			}
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"news" => []
-		];
-		
-		// load html
-		$this->fuckhtml->load($html);
-		
-		$this->detect_captcha($html);
-		
-		// get npt
-		$out["npt"] =
-			$this->generatenextpagetoken(
-				$search,
-				$nsfw,
-				$country,
-				$spellcheck,
-				"news",
-				$proxy
-			);
-		
-		$this->fuckhtml->load($html);
-		$jsons = $this->get_js();
-		
-		foreach($jsons as $json){
-			foreach(
-				$json["data"]["response"]["news"]["results"]
-				as $news
-			){
-				
-				if(
-					!isset($news["thumbnail"]["src"]) ||
-					$news["thumbnail"]["src"] == "void 0"
-				){
-									
-					$thumb = [
-						"url" => null,
-						"ratio" => null
-					];
-				}else{
-									
-					$thumb = [
-						"url" => $this->unshiturl($news["thumbnail"]["src"]),
-						"ratio" => "16:9"
-					];
-				}
-				
-				$out["news"][] = [
-					"title" => $news["title"],
-					"author" => null,
-					"description" => $news["description"],
-					"date" => !isset($news["age"]) || $news["age"] == "void 0" || $news["age"] == "null" ? null : strtotime($news["age"]),
-					"thumb" => $thumb,
-					"url" => $news["url"]
-				];
-			}
-		}
-		
-		return $out;
-	}
-	
 	public function image($get){
 		
 		$search = $get["s"];
@@ -1298,6 +1172,12 @@ class brave{
 		$jsons = $this->get_js();
 		
 		foreach($jsons as $json){
+			
+			if(!isset($json["data"]["body"]["response"]["results"])){
+				
+				continue;
+			}
+			
 			foreach(
 				$json
 				["data"]
@@ -1325,7 +1205,7 @@ class brave{
 				];
 			}
 		}
-			
+		
 		return $out;
 	}
 	
@@ -1431,8 +1311,15 @@ class brave{
 		
 		$this->fuckhtml->load($html);
 		$jsons = $this->get_js();
+		$error = false;
 		
 		foreach($jsons as $json){
+			
+			if(!isset($json["data"]["body"]["response"]["results"])){
+				
+				continue;
+			}
+			
 			foreach(
 				$json
 				["data"]
@@ -1488,6 +1375,147 @@ class brave{
 		return $out;
 	}
 	
+	public function news($get){
+		
+		if($get["npt"]){
+			
+			[$req, $proxy] = $this->backend->get($get["npt"], "news");
+			
+			$req = json_decode($req, true);
+			
+			$search = $req["q"];
+			$country = $req["country"];
+			$nsfw = $req["nsfw"];
+			$offset = $req["offset"];
+			$spellcheck = $req["spellcheck"];
+			
+			try{
+				$html =
+					$this->get(
+						$proxy,
+						"https://search.brave.com/news",
+						[
+							"q" => $search,
+							"offset" => $offset,
+							"spellcheck" => $spellcheck
+						],
+						$nsfw,
+						$country
+					);
+				
+			}catch(Exception $error){
+				
+				throw new Exception("Could not fetch search page");
+			}
+			
+		}else{
+			$search = $get["s"];
+			if(strlen($search) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			if(strlen($search) > 2048){
+				
+				throw new Exception("Search term is too long!");
+			}
+			
+			$proxy = $this->backend->get_ip();
+			$nsfw = $get["nsfw"];
+			$country = $get["country"];
+			$spellcheck = $get["spellcheck"] == "yes" ? "1" : "0";
+			
+			/*
+			$handle = fopen("scraper/brave-news.html", "r");
+			$html = fread($handle, filesize("scraper/brave-news.html"));
+			fclose($handle);*/
+			try{
+				$html =
+					$this->get(
+						$proxy,
+						"https://search.brave.com/news",
+						[
+							"q" => $search,
+							"spellcheck" => $spellcheck
+						],
+						$nsfw,
+						$country
+					);
+				
+			}catch(Exception $error){
+				
+				throw new Exception("Could not fetch search page");
+			}
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"news" => []
+		];
+		
+		// load html
+		$this->fuckhtml->load($html);
+		
+		$this->detect_captcha($html);
+		
+		// get npt
+		$out["npt"] =
+			$this->generatenextpagetoken(
+				$search,
+				$nsfw,
+				$country,
+				$spellcheck,
+				"news",
+				$proxy
+			);
+		
+		$this->fuckhtml->load($html);
+		$jsons = $this->get_js();
+		
+		foreach($jsons as $json){
+						
+			if(!isset($json["data"]["response"]["news"]["results"])){
+				
+				continue;
+			}
+			
+			foreach(
+				$json["data"]["response"]["news"]["results"]
+				as $news
+			){
+				
+				if(
+					!isset($news["thumbnail"]["src"]) ||
+					$news["thumbnail"]["src"] == "void 0"
+				){
+									
+					$thumb = [
+						"url" => null,
+						"ratio" => null
+					];
+				}else{
+									
+					$thumb = [
+						"url" => $this->unshiturl($news["thumbnail"]["src"]),
+						"ratio" => "16:9"
+					];
+				}
+				
+				$out["news"][] = [
+					"title" => $news["title"],
+					"author" => null,
+					"description" => $news["description"],
+					"date" => !isset($news["age"]) || $news["age"] == "void 0" || $news["age"] == "null" ? null : strtotime($news["age"]),
+					"thumb" => $thumb,
+					"url" => $news["url"]
+				];
+			}
+		}
+		
+		return $out;
+	}
+	
 	private function stackoverflow_parse($html){
 		
 		$i = 0;

+ 1 - 1
scraper/google.php

@@ -817,7 +817,7 @@ class google{
 			// country
 			if($get["country"] != "any"){
 				
-				$params["gl"] = $country;
+				$params["gl"] = $get["country"];
 			}
 			
 			// nsfw

+ 2 - 0
scraper/google_api.php

@@ -567,6 +567,8 @@ class google_api{
 		
 		foreach($json["items"] as $result){
 			
+			$date = null;
+			
 			//
 			// probe for thumbnail
 			//

+ 10 - 1
scraper/yt.php

@@ -678,6 +678,15 @@ class yt{
 							$video["longBylineText"]
 						),
 					"url" =>
+						isset(
+							$video["ownerText"]
+							["runs"]
+							[0]
+							["navigationEndpoint"]
+							["commandMetadata"]
+							["webCommandMetadata"]
+							["url"]
+						) ?
 						"https://www.youtube.com" .
 						$video["ownerText"]
 						["runs"]
@@ -685,7 +694,7 @@ class yt{
 						["navigationEndpoint"]
 						["commandMetadata"]
 						["webCommandMetadata"]
-						["url"],
+						["url"] : null,
 					"avatar" =>
 						$video["channelThumbnailSupportedRenderers"]
 						["channelThumbnailWithLinkRenderer"]