1
0
Эх сурвалжийг харах

scrape google sublinks, which are encrypted pieces of shit

lolcat 2 долоо хоног өмнө
parent
commit
90232152e9
1 өөрчлөгдсөн 44 нэмэгдсэн , 2 устгасан
  1. 44 2
      scraper/google.php

+ 44 - 2
scraper/google.php

@@ -733,7 +733,7 @@ class google{
 			$container = &$data["container"];
 			$container = &$data["container"];
 			
 			
 			//file_put_contents("scraper/google-jp.html", $html);
 			//file_put_contents("scraper/google-jp.html", $html);
-			//$html = file_get_contents("scraper/dmca.html");
+			//$html = file_get_contents("scraper/sublink.html");
 			//$container = "";
 			//$container = "";
 		}
 		}
 		
 		
@@ -772,9 +772,10 @@ class google{
 			$html = &$data2["data"];
 			$html = &$data2["data"];
 						
 						
 			$this->fuckhtml->load($html);
 			$this->fuckhtml->load($html);
-			$this->parsestyles();
 		}
 		}
 		
 		
+		$this->parsestyles();
+		
 		$this->scrape_dimg($html);
 		$this->scrape_dimg($html);
 		$this->scrape_imagearr($html);
 		$this->scrape_imagearr($html);
 		
 		
@@ -1269,6 +1270,47 @@ class google{
 			
 			
 			$sublinks = [];
 			$sublinks = [];
 			
 			
+			// get sublinks (rare)
+			$as =
+				$this->fuckhtml
+				->getElementsByClassName(
+					$this->getstyle([
+						"padding-right" => "6px",
+						"margin-top" => "4px",
+						"display" => "inline-block"
+					]),
+					"a"
+				);
+			
+			foreach($as as $sublink){
+				
+				$sublink_url =
+					$this->fuckhtml
+					->getTextContent(
+						$sublink["attributes"]["href"]
+					);
+				
+				if(
+					preg_match(
+						'/^\/goto\?url=(.*)/',
+						$sublink_url,
+						$piece_of_shit
+					)
+				){
+					
+					// encrypted piece of shit url
+					$sublink_url = "/resolver?scraper=google&target=" . $piece_of_shit[1];
+				}
+				
+				$sublinks[] = [
+					"title" =>
+						$this->fuckhtml
+						->getTextContent($sublink),
+					"date" => null,
+					"url" => $sublink_url
+				];
+			}
+			
 			$out["web"][] = [
 			$out["web"][] = [
 				"title" => $title,
 				"title" => $title,
 				"description" => $description,
 				"description" => $description,