4get-mirror/lib/backend.php

<?php
class backend{
	
	public function __construct($scraper){
		
		$this->scraper = $scraper;
		$this->db = null;
		$this->db_path_prefix = "data/sessions/";
		$this->db_path = "{$this->db_path_prefix}{$scraper}.sqlite";
	}
	
	/*
		Proxy stuff
	*/
	public function get_ip($proxy_index_raw = null){
		
		$pool = constant("config::PROXY_" . strtoupper($this->scraper));
		if($pool === false){
			
			// we don't want a proxy, fuck off!
			return 'raw_ip::::';
		}
		
		// indent
		if($proxy_index_raw === null){
			
			$proxy_index_raw = apcu_inc("p." . $this->scraper);
		}
		
		$proxylist = file_get_contents("data/proxies/" . $pool . ".txt");
		$proxylist = explode("\n", $proxylist);

		// ignore empty or commented lines
		$proxylist = array_filter($proxylist, function($entry){
			$entry = ltrim($entry);
			return strlen($entry) > 0 && substr($entry, 0, 1) != "#";
		});
		
		$proxylist = array_values($proxylist);
		
		if(count($proxylist) === 0){
			
			throw new Exception("A proxy list was specified but it's empty!");
		}
		
		//echo $proxylist[$proxy_index_raw % count($proxylist)];
		return $proxylist[$proxy_index_raw % count($proxylist)];
	}
	
	// this function is also called directly on nextpage
	public function assign_proxy(&$curlproc, string $ip){
		
		// parse proxy line
		[
			$type,
			$address,
			$port,
			$username,
			$password
		] = explode(":", $ip, 5);
		
		switch($type){
			
			case "raw_ip":
				return;
				break;
			
			case "http":
			case "https":
				curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
				curl_setopt($curlproc, CURLOPT_PROXY, $type . "://" . $address . ":" . $port);
				break;
			
			case "socks4":
				curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4);
				curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port);
				break;
			
			case "socks5":
				curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
				curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port);
				break;
			
			case "socks4a":
				curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4A);
				curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port);
				break;
			
			case "socks5_hostname":
			case "socks5h":
			case "socks5a":
				curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME);
				curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port);
				break;
		}
		
		if($username != ""){
			
			curl_setopt($curlproc, CURLOPT_PROXYUSERPWD, $username . ":" . $password);
		}
	}
	
	// API key rotation
	public function get_key(){
		
		$keys = file_get_contents("data/api_keys/" . $this->scraper . ".txt");
		$keys = explode("\n", $keys);
		
		$keys = array_filter($keys, function($entry){
			$entry = ltrim($entry);
			return strlen($entry) > 0 && substr($entry, 0, 1) != "#";
		});
		
		$keys = array_values($keys);
		
		if(count($keys) === 0){
			
			throw new Exception("Please specify API keys in data/api_keys/" . $this->scraper . ".txt");
		}
		
		$increment = apcu_inc("s." . $this->scraper) % count($keys);
		return [
			"key" => $keys[$increment],
			"increment" => $increment
		];
	}
	
	
	/*
		Next page stuff
	*/
	public function store(string $payload, string $page, string $proxy){
		
		$key = sodium_crypto_secretbox_keygen();
		$nonce = random_bytes(SODIUM_CRYPTO_SECRETBOX_NONCEBYTES);
		
		$requestid = apcu_inc("requestid");
		
		apcu_store(
			$page[0] . "." . // first letter of page name
			$this->scraper . // scraper name
			$requestid,
			[
				$nonce,
				$proxy,
				// compress and encrypt
				sodium_crypto_secretbox(
					gzdeflate($payload),
					$nonce,
					$key
				)
			],
			900 // cache information for 15 minutes
		);

		return 
			$this->scraper . $requestid . "." .
			rtrim(strtr(base64_encode($key), '+/', '-_'), '=');
	}
	
	public function get(string $npt, string $page){
		
		$page = $page[0];
		$explode = explode(".", $npt, 2);
		
		if(count($explode) !== 2){
			
			throw new Exception("Malformed nextPageToken!");
		}
		
		$apcu = $page . "." . $explode[0];
		$key = $explode[1];
		
		$payload = apcu_fetch($apcu);
		
		if($payload === false){
			
			throw new Exception("The next page token is invalid or has expired!");
		}
		
		$key =
			base64_decode(
				str_pad(
					strtr($key, '-_', '+/'),
					strlen($key) % 4,
					'=',
					STR_PAD_RIGHT
				)
			);
		
		// decrypt and decompress data
		$payload[2] =
			gzinflate(
				sodium_crypto_secretbox_open(
					$payload[2], // data
					$payload[0], // nonce
					$key
				)
			);
		
		if($payload[2] === false){
			
			throw new Exception("The next page token is invalid or has expired!");
		}
		
		// remove the key after using successfully
		apcu_delete($apcu);
		
		return [
			$payload[2], // data
			$payload[1] // proxy
		];
	}
	
	//
	// Sessions
	//
	public function session_init_db($force_init = false){
		
		if(
			$force_init ||
			$this->db === null
		){
			
			$this->db = new SQLite3($this->db_path);
			
			// create table if it doesnt exist	
			$this->db->exec(
				"CREATE TABLE IF NOT EXISTS sessions ( " .
					"id INTEGER PRIMARY KEY AUTOINCREMENT, " .
					"created_at INTEGER NOT NULL, " .
					"proxy TEXT NOT NULL, " .
					"headers_json TEXT NOT NULL" .
				")"
			);
			
			return true;
		}
		
		return false;
	}
	
	public function session_count(){
		
		$this->session_init_db();
		
		$stmt = $this->db->prepare(
			"SELECT COUNT(*) as count FROM sessions"
		);
		
		$result = $stmt->execute();
		$row = $result->fetchArray(SQLITE3_ASSOC);
		
		return (int)($row["count"] ?? 0);
	}
	
	public function session_write($proxy, $headers){
		
		$this->session_init_db();
		
		$stmt = $this->db->prepare(
			"INSERT INTO sessions ( " .
				"created_at, " .
				"proxy, " .
				"headers_json " .
			") VALUES ( " .
				":created_at, " .
				":proxy, " .
				":headers_json" .
			")"
		);
		
		$stmt->bindValue(':created_at', time(), SQLITE3_INTEGER);
		$stmt->bindValue(':proxy', $proxy, SQLITE3_TEXT);
		$stmt->bindValue(':headers_json', json_encode($headers), SQLITE3_TEXT);
		
		$stmt->execute();
		
		return $this->db->lastInsertRowID();
	}
	
	public function session_get(){
		
		$count = $this->session_count();
		
		if($count === 0){
			
			return false;
		}
		
		$offset = apcu_inc("d." . $this->scraper) % $count;
		
		$stmt = $this->db->prepare(
			"SELECT * " .
			"FROM sessions " .
			"ORDER BY id ASC " .
			"LIMIT 1 OFFSET :offset"
		);
		
		$stmt->bindValue(':offset', $offset, SQLITE3_INTEGER);
		
		$result = $stmt->execute();
		$row = $result->fetchArray(SQLITE3_ASSOC);
		
		return [
			"id" => $row["id"],
			"created_at" => $row["created_at"],
			"proxy" => $row["proxy"],
			"headers" => json_decode($row["headers_json"], true)
		];
	}
	
	public function session_destroy_list(){
		
		if(
			!preg_match(
				'/^[A-Za-z0-9-]+$/',
				$this->scraper
			)
		){
			
			// path traversal
			return false;
		}
		
		if(
			file_exists("{$this->db_path_prefix}{$this->scraper}.sqlite") &&
			is_file("{$this->db_path_prefix}{$this->scraper}.sqlite")
		){
			
			unlink("{$this->db_path_prefix}{$this->scraper}.sqlite");
			return true;
		}
		
		return false;
	}
}