浏览代码

duckduckgo scraper rewrite

lolcat 1 年之前
父节点
当前提交
774f7113df
共有 6 个文件被更改,包括 1026 次插入1555 次删除
  1. 14 7
      api.txt
  2. 1 1
      data/config.php
  3. 1 0
      lib/backend.php
  4. 81 0
      lib/fuckhtml.php
  5. 929 1547
      scraper/ddg.php
  6. 0 0
      scraper/out.txt

+ 14 - 7
api.txt

@@ -1,10 +1,17 @@
-                        __ __             __
-                       / // / ____ ____  / /_
-                      / // /_/ __ `/ _ \/ __/
-                     /__  __/ /_/ /  __/ /_ 
-                       /_/  \__, /\___/\__/
-                           /____/         
-
+                   44
+                 4444444      44   
+                 44444444   44444       444
+                 44444444  444444    444444444
+                  44444   44444444  444444444
+                         444444444   4444444
+                        4444444444    444444
+                      4444444444444
+                    444444444444444444
+                       444444444444444
+                          44444444
+                          4444
+                           44
+                 
            + Welcome to the 4get API documentation +
 
 + Terms of use

+ 1 - 1
data/config.php

@@ -119,7 +119,7 @@ class config{
 	
 	// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
 	// Changing this might break things.
-	const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0";
+	const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0";
 	
 	// Proxy pool assignments for each scraper
 	// false = Use server's raw IP

+ 1 - 0
lib/backend.php

@@ -75,6 +75,7 @@ class backend{
 				break;
 			
 			case "socks5_hostname":
+			case "socks5h":
 			case "socks5a":
 				curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME);
 				curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port);

+ 81 - 0
lib/fuckhtml.php

@@ -526,4 +526,85 @@ class fuckhtml{
 				$string
 			);
 	}
+	
+	public function extract_json($json){
+		
+		$len = strlen($json);
+		$array_level = 0;
+		$object_level = 0;
+		$in_quote = null;
+		$start = null;
+		
+		for($i=0; $i<$len; $i++){
+			
+			switch($json[$i]){
+				
+				case "[":
+					if($in_quote === null){
+						
+						$array_level++;
+						if($start === null){
+							
+							$start = $i;
+						}
+					}
+					break;
+				
+				case "]":
+					if($in_quote === null){
+						
+						$array_level--;
+					}
+					break;
+				
+				case "{":
+					if($in_quote === null){
+						
+						$object_level++;
+						if($start === null){
+							
+							$start = $i;
+						}
+					}
+					break;
+				
+				case "}":
+					if($in_quote === null){
+						
+						$object_level--;
+					}
+					break;
+				
+				case "\"":
+				case "'":
+					if(
+						$i !== 0 &&
+						$json[$i - 1] !== "\\"
+					){
+						// found a non-escaped quote
+						
+						if($in_quote === null){
+							
+							// open quote
+							$in_quote = $json[$i];
+						}elseif($in_quote === $json[$i]){
+							
+							// close quote
+							$in_quote = null;
+						}
+					}
+					break;
+			}
+			
+			if(
+				$start !== null &&
+				$array_level === 0 &&
+				$object_level === 0
+			){
+				
+				return substr($json, $start, $i - $start + 1);
+				break;
+			}
+		}
+	}
 }

文件差异内容过多而无法显示
+ 929 - 1547
scraper/ddg.php


文件差异内容过多而无法显示
+ 0 - 0
scraper/out.txt


部分文件因为文件数量过多而无法显示