google scraper fix

data/config.php

@@ -24,15 +24,21 @@ class config{
 	const API_ENABLED = true;
 	
 	//
-	// 4play (session provider)
+	// 4play settings
 	//
-	// Enable 4play API?
-	const FPLAY_ENABLE_API = true;
+	// Enable 4get's /api/v2/provide_sesh endpoint
+	const FPLAY_ENABLE_API = false;
 	
-	// 4play password. Please set this to something secure if you enable the 4play API.
+	// 4play password. Please set this to something secure if you use this extension.
 	// This password is used to POST sessions to /api/v2/provide_sesh
+	// It's also used as the external endpoint password.
 	const FPLAY_PASSWORD = "cnc";
 	
+	// Endpoint 4get hits to render webpages
+	// Eg: null=no endpoint provided
+	// Or "https://your-endpoint.com/my-endpoint" (excluding get parameters like &pw)
+	const FPLAY_EXTERNAL_ENDPOINT = null;
+	
 	//
 	// BOT PROTECTION
 	//

docs/configure.md

@@ -79,3 +79,16 @@ Done! The scraper you chose should now be using the rotating proxies. When askin
 
 ## Important!
 If you ever test out a `socks5` proxy locally on your machine and find out it works but doesn't on your server, try supplying the `socks5_hostname` protocol instead. Hopefully this tip can save you 3 hours of your life!
+
+# 4play
+4play is a botfaggotry Firefox extension. I made it, and it allows me to control a Firefox browser through code. More information about the project [here](https://git.lolcat.ca/lolcat/4play). It allows you to scrape Google (& probably more sites in the future)
+
+1. Install the [firefox extension](https://addons.mozilla.org/en-US/firefox/addon/4play/) onto a clean Firefox profile. Make sure you have a real display connected and that you use real hardware with an user-grade GPU. Websites detect this shit now.
+2. Navigate to `/extra/4play/`
+3. Install the 4play server: `npm install @lawlers/4play`
+4. Edit the `render-server.js` file. Change the line `var password = "cnc";` so that it uses a secure password.
+5. Edit `/data/config.php` and update the 4play password. Also update the `FPLAY_EXTERNAL_ENDPOINT` so that it points to the render HTTP server.
+
+Now, if you did everything correctly, you should be able to render search pages on scrapers that require it. If your render server sits on a different server, I highly recommend setting up a reverse proxy to add TLS support.
+
+In my case, the 4get website & scrapers sits at the OVH datacenter, while the render server is at my home running on a mini-PC I bought for the occasion. This wouldn't have been possible without your financial support, thank you all!!

extra/4play/page-render.js

@@ -0,0 +1,341 @@
+const fplay = require("@lawlers/4play");
+const http = require("http");
+
+// 4play settings
+var port = 3030;
+var timeout = 30000;
+var password = "cnc";
+
+// ==================================================
+
+// webserver
+
+var global_ws = null;
+const webserver_port = 3000;
+
+var callback_q = new Map();
+
+const server = http.createServer(async function(req, res){
+	
+	const parsed = new URL(req.url, "http://localhost");
+	
+	const url = parsed.searchParams.get("url");
+	const pw = parsed.searchParams.get("pw");
+	var proxy = parsed.searchParams.get("proxy");
+	const url_match = parsed.searchParams.get("url_match");
+	const content_match = parsed.searchParams.get("content_match");
+	const fail_url_match = parsed.searchParams.get("fail_url_match");
+	const fail_content_match = parsed.searchParams.get("fail_content_match");
+	
+	var container = parsed.searchParams.get("container");
+	
+	if(fplay.browser_connected === false){
+				
+		res.writeHead(
+			400,
+			{
+				"Content-Type": "text/plain"
+			}
+		);
+		
+		res.end("No browser available to render the motherfucking page");
+		return;
+	}
+	
+	var errors = [];
+	
+	if(
+		pw === null ||
+		pw != password
+	){
+		
+		errors.push("pw");
+	}
+	
+	if(url === null){ errors.push("url"); }
+	if(proxy === null){ errors.push("proxy"); }
+	
+	if(errors.length !== 0){
+		
+		res.writeHead(
+			400,
+			{
+				"Content-Type": "text/plain"
+			}
+		);
+		
+		res.end("The following parameters are missing or invalid: " + errors.join(", "));
+		return;
+	}
+	
+	console.log("Rendering " + url);
+	
+	if(
+		container === null ||
+		await fplay.container_exists(global_ws, container) === false
+	){
+		
+		container = await fplay.container_create(global_ws);
+		
+		proxy = get_proxy(proxy);
+		
+		// assign proxy
+		if(proxy.config !== false){
+			
+			await fplay.container_attach_proxy(
+				global_ws,
+				container,
+				proxy.config
+			);
+		}
+	}else{
+		
+		proxy = {raw: proxy};
+	}
+	
+	// open tab
+	const newtab = await fplay.tab_open(global_ws, url, false, container);
+	
+	if(newtab === false){
+		
+		// failed to open tab
+		res.writeHead(
+			400,
+			{
+				"Content-Type": "text/plain",
+				"X-Proxy": proxy.raw,
+				"X-Container-ID": typeof container.id == "undefined" ? container : container.id
+			}
+		);
+		
+		res.end("Failed to open tab");
+		return;
+	}
+		
+	// callback will be handled in "web_response"
+	callback_q.set(
+		newtab.id,
+		{
+			http_res: res,
+			proxy: proxy,
+			container: container,
+			url_match: url_match,
+			content_match: content_match,
+			fail_url_match: fail_url_match,
+			fail_content_match: fail_content_match
+		}
+	);
+	
+	// close tab in 30 seconds
+	await fplay.wait_random(15000, 35000);
+	await fplay.tab_close(global_ws, newtab);
+});
+
+server.listen(webserver_port, function(){
+	
+	console.log("http server running");
+});
+
+function get_proxy(parts){
+	
+	parts = parts.split(":");
+	// format: type:address:port:username:password
+	// PHP explode(":", $ip, 5) splits into max 5 parts
+	var type = parts[0];
+	const address = parts[1];
+	const port = parts[2];
+	const username = parts[3];
+	const password = parts[4];
+	
+	var proxy_dns = true;
+	
+	switch(type){
+		case "raw_ip":
+		case "ip":
+			// no proxy, use direct connection
+			return {
+				config: false,
+				raw: "raw_ip::::"
+			}
+			break;
+		
+		case "socks5":
+			type = "socks";
+			proxy_dns = false;
+			break;
+			
+		case "socks4a":
+			type = "socks4";
+			break;
+			
+		case "socks4":
+			proxy_dns = false;
+			break;
+			
+		case "socks5_hostname":
+		case "socks5h":
+		case "socks5a":
+			type = "socks";
+			break;
+		
+		case "http":
+		case "https":
+			// do nothing
+			break;
+		
+		default:
+			// invalid proxy provided
+			return {
+				config: false,
+				raw: "raw_ip::::"
+			}
+	}
+	
+	return {
+		config: {
+			type: type,
+			host: address,
+			port: port,
+			username: username,
+			password: password,
+			proxyDNS: proxy_dns
+		},
+		raw: proxy_line
+	};
+}
+
+fplay.event.on("server_ready", function(){
+	
+	console.log("listening on port " + port + " (timeout=" + timeout + ")");
+});
+
+fplay.event.on("browser_connect", async function(ws){
+	
+	global_ws = ws;
+	
+	const ua = await fplay.get_ua(ws);
+	console.log("New browser instance connected: " + ua);
+	
+	await fplay.close_all_tabs(ws);
+	await fplay.delete_all_containers(ws);
+	
+	fplay.web_response_whitelist(ws, ["main_frame"]);
+	
+	// clean up
+	const blanktab = await fplay.close_all_tabs(ws);
+	await fplay.delete_all_containers(ws);
+});
+
+fplay.event.on("web_response", async function(page){
+	
+	var callback_data = callback_q.get(page.id);
+	
+	if(typeof callback_data == "undefined"){ return; }
+	
+	var body = page.body.toString();
+	
+	var url_match = page.url.match(new RegExp(callback_data.url_match, "i"));
+	var content_match = body.match(new RegExp(callback_data.content_match, "i"));
+	var fail_url_match = page.url.match(new RegExp(callback_data.fail_url_match, "i"));
+	var fail_content_match = body.match(new RegExp(callback_data.fail_content_match, "i"));
+	
+	// handle positive response
+	if(
+		(
+			callback_data.url_match === null &&
+			callback_data.content_match === null
+		) ||
+		(
+			url_match &&
+			content_match
+		) ||
+		(
+			url_match &&
+			callback_data.content_match === null
+		) ||
+		(
+			callback_data.url_match === null &&
+			content_match
+		)
+	){
+		
+		callback_data.http_res.writeHead(
+			200,
+			{
+				"Content-Type": "text/plain;charset=UTF-8",
+				"X-Proxy": callback_data.proxy.raw,
+				"X-Container-ID": typeof callback_data.container.id == "undefined" ? callback_data.container : callback_data.container.id
+			}
+		);
+		
+		callback_data.http_res.end(body);
+		
+		callback_q.delete(page.id); // cleanup callback only
+		return;
+	}
+	
+	// handle negative response
+	if(
+		(
+			callback_data.fail_url_match !== null &&
+			callback_data.fail_content_match !== null
+		) &&
+		(
+			(
+				fail_url_match &&
+				fail_content_match
+			) ||
+			(
+				fail_url_match &&
+				callback_data.fail_content_match === null
+			) ||
+			(
+				callback_data.fail_url_match === null &&
+				fail_content_match
+			)
+		)
+	){
+		
+		callback_data.http_res.writeHead(
+			400,
+			{
+				"Content-Type": "text/plain",
+				"X-Proxy": callback_data.proxy.raw,
+				"X-Container-ID": callback_data.container.id
+			}
+		);
+		
+		callback_data.http_res.end("Reached fail state (matched fail_url_match or fail_content_match)");
+		
+		// cleanup browser + callback
+		await fplay.tab_close(global_ws, page.id);
+		await fplay.container_delete(global_ws, callback_data.container);
+		callback_q.delete(page.id);
+	}
+});
+
+fplay.event.on("dom_load_fail", async function(page){
+	
+	var callback_data = callback_q.get(page.id);
+	
+	if(typeof callback_data == "undefined"){ return; }
+	
+	callback_q.delete(page.id);
+	
+	callback_data.http_res.writeHead(
+		400,
+		{
+			"Content-Type": "text/plain",
+			"X-Proxy": callback_data.proxy.raw,
+			"X-Container-ID": typeof callback_data.container.id == "undefined" ? callback_data.container : callback_data.container.id
+		}
+	);
+	
+	callback_data.http_res.end("Network error (" + page.error + ")");
+		
+	// cleanup browser
+	await fplay.tab_close(global_ws, page.id);
+	await fplay.container_delete(global_ws, callback_data.container);
+});
+
+fplay.init(port, password, timeout);

lib/backend.php

@@ -4,6 +4,9 @@ class backend{
 	public function __construct($scraper){
 		
 		$this->scraper = $scraper;
+		$this->db = null;
+		$this->db_path_prefix = "data/sessions/";
+		$this->db_path = "{$this->db_path_prefix}{$scraper}.sqlite";
 	}
 	
 	/*
@@ -208,4 +211,127 @@ class backend{
 			$payload[1] // proxy
 		];
 	}
+	
+	//
+	// Sessions
+	//
+	public function session_init_db($force_init = false){
+		
+		if(
+			$force_init ||
+			$this->db === null
+		){
+			
+			$this->db = new SQLite3($this->db_path);
+			
+			// create table if it doesnt exist	
+			$this->db->exec(
+				"CREATE TABLE IF NOT EXISTS sessions ( " .
+					"id INTEGER PRIMARY KEY AUTOINCREMENT, " .
+					"created_at INTEGER NOT NULL, " .
+					"proxy TEXT NOT NULL, " .
+					"headers_json TEXT NOT NULL" .
+				")"
+			);
+			
+			return true;
+		}
+		
+		return false;
+	}
+	
+	public function session_count(){
+		
+		$this->session_init_db();
+		
+		$stmt = $this->db->prepare(
+			"SELECT COUNT(*) as count FROM sessions"
+		);
+		
+		$result = $stmt->execute();
+		$row = $result->fetchArray(SQLITE3_ASSOC);
+		
+		return (int)($row["count"] ?? 0);
+	}
+	
+	public function session_write($proxy, $headers){
+		
+		$this->session_init_db();
+		
+		$stmt = $this->db->prepare(
+			"INSERT INTO sessions ( " .
+				"created_at, " .
+				"proxy, " .
+				"headers_json " .
+			") VALUES ( " .
+				":created_at, " .
+				":proxy, " .
+				":headers_json" .
+			")"
+		);
+		
+		$stmt->bindValue(':created_at', time(), SQLITE3_INTEGER);
+		$stmt->bindValue(':proxy', $proxy, SQLITE3_TEXT);
+		$stmt->bindValue(':headers_json', json_encode($headers), SQLITE3_TEXT);
+		
+		$stmt->execute();
+		
+		return $this->db->lastInsertRowID();
+	}
+	
+	public function session_get(){
+		
+		$count = $this->session_count();
+		
+		if($count === 0){
+			
+			return false;
+		}
+		
+		$offset = apcu_inc("d." . $this->scraper) % $count;
+		
+		$stmt = $this->db->prepare(
+			"SELECT * " .
+			"FROM sessions " .
+			"ORDER BY id ASC " .
+			"LIMIT 1 OFFSET :offset"
+		);
+		
+		$stmt->bindValue(':offset', $offset, SQLITE3_INTEGER);
+		
+		$result = $stmt->execute();
+		$row = $result->fetchArray(SQLITE3_ASSOC);
+		
+		return [
+			"id" => $row["id"],
+			"created_at" => $row["created_at"],
+			"proxy" => $row["proxy"],
+			"headers" => json_decode($row["headers_json"], true)
+		];
+	}
+	
+	public function session_destroy_list(){
+		
+		if(
+			!preg_match(
+				'/^[A-Za-z0-9-]+$/',
+				$this->scraper
+			)
+		){
+			
+			// path traversal
+			return false;
+		}
+		
+		if(
+			file_exists("{$this->db_path_prefix}{$this->scraper}.sqlite") &&
+			is_file("{$this->db_path_prefix}{$this->scraper}.sqlite")
+		){
+			
+			unlink("{$this->db_path_prefix}{$this->scraper}.sqlite");
+			return true;
+		}
+		
+		return false;
+	}
 }