| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162 |
- <?php
- class yahoo_japan{
-
- public function __construct(){
-
- include "lib/backend.php";
- $this->backend = new backend("yahoo_japan");
-
- include "lib/fuckhtml.php";
- $this->fuckhtml = new fuckhtml();
- }
-
- public function getfilters($page){
-
- return [];
- }
-
- private function get($proxy, $url, $get = [], $return_cookies = false, $is_xhr = false, $cookie = null){
-
- $curlproc = curl_init();
-
- if($get !== []){
- $get = http_build_query($get);
- $url .= "?" . $get;
- }
-
- curl_setopt($curlproc, CURLOPT_URL, $url);
-
- // http2 bypass
- curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
-
- curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-
- if($cookie !== null){
-
- $c = [];
- foreach($cookie as $name => $value){
-
- $c[] = "{$name}=$value";
- }
-
- $cookie = implode("; ", $c);
- }
-
- if($is_xhr){
-
- curl_setopt($curlproc, CURLOPT_HTTPHEADER,
- ["User-Agent: " . config::USER_AGENT,
- "Accept: application/json, text/plain, */*",
- "Accept-Language: en-US,en;q=0.5",
- "Accept-Encoding: gzip",
- "Referer: https://search.yahoo.co.jp/",
- "DNT: 1",
- "Sec-GPC: 1",
- "Connection: keep-alive",
- "Cookie: " . $cookie,
- "Sec-Fetch-Dest: empty",
- "Sec-Fetch-Mode: cors",
- "Sec-Fetch-Site: same-origin",
- "TE: trailers"]
- );
- }else{
-
- curl_setopt($curlproc, CURLOPT_HTTPHEADER,
- ["User-Agent: " . config::USER_AGENT,
- "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
- "Accept-Language: en-US,en;q=0.5",
- "Accept-Encoding: gzip",
- "DNT: 1",
- "Sec-GPC: 1",
- "Connection: keep-alive",
- "Upgrade-Insecure-Requests: 1",
- "Sec-Fetch-Dest: document",
- "Sec-Fetch-Mode: navigate",
- "Sec-Fetch-Site: same-origin",
- "Sec-Fetch-User: ?1",
- "Priority: u=0, i",
- "TE: trailers"]
- );
- }
-
- curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
- curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
- curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
- curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
- curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-
- $this->backend->assign_proxy($curlproc, $proxy);
-
- if($return_cookies){
-
- // extract cookies
- $cookies_tmp = [];
- curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies_tmp){
-
- $length = strlen($header);
-
- $header = explode(":", $header, 2);
-
- if(trim(strtolower($header[0])) == "set-cookie"){
-
- $cookie_tmp = explode("=", trim($header[1]), 2);
-
- $cookies_tmp[trim($cookie_tmp[0])] =
- explode(";", $cookie_tmp[1], 2)[0];
- }
-
- return $length;
- });
- }
-
- $data = curl_exec($curlproc);
-
- if(curl_errno($curlproc)){
-
- throw new Exception(curl_error($curlproc));
- }
-
- curl_close($curlproc);
-
- if($return_cookies){
-
- return [
- "cookies" => $cookies_tmp,
- "body" => $data
- ];
- }
-
- return $data;
- }
-
- public function web($get){
-
- if($get["npt"]){
-
- [$url, $proxy] = $this->backend->get($get["npt"], "web");
- $params = [];
-
- }else{
-
- $search = $get["s"];
- if(strlen($search) === 0){
-
- throw new Exception("Search term is empty!");
- }
-
- $proxy = $this->backend->get_ip();
-
- $url = "https://search.yahoo.co.jp/search";
- $params = [
- "p" => $get["s"]
- ];
- }
-
- try{
- $html = $this->get(
- $proxy,
- $url,
- $params
- );
- }catch(Exception $error){
-
- throw new Exception("Failed to fetch search page");
- }
- //$html = file_get_contents("scraper/yahoo_japan.html");
-
- $out = [
- "status" => "ok",
- "spelling" => [
- "type" => "no_correction",
- "using" => null,
- "correction" => null
- ],
- "npt" => null,
- "answer" => [],
- "web" => [],
- "image" => [],
- "video" => [],
- "news" => [],
- "related" => []
- ];
-
- $json_object =
- explode(
- '<script id="__NEXT_DATA__" type="application/json">',
- $html
- );
-
- if(count($json_object) !== 2){
-
- throw new Exception("Failed to find JSON script");
- }
-
- $json =
- json_decode(
- $this->fuckhtml
- ->extract_json(
- $json_object[1]
- ),
- true
- );
-
- if($json === null){
-
- throw new Exception("Failed to decode JSON");
- }
-
- //print_r($json);
-
- //
- // Extract mainline search results
- //
- if(!isset($json["props"]["pageProps"]["initialProps"]["pageData"]["algos"])){
-
- throw new Exception("Failed to access algos object");
- }
-
- foreach($json["props"]["pageProps"]["initialProps"]["pageData"]["algos"] as $result){
-
- switch($result["type"]){
-
- case "Algo":
- if(isset($result["visualWebImageGallery"]["imageThumbs"][0]["source"])){
-
- $thumb = [
- "ratio" => "1:1",
- "url" => $result["visualWebImageGallery"]["imageThumbs"][0]["source"]
- ];
- }elseif(isset($result["visualWebImageSnippet"])){
-
- $thumb = [
- "ratio" => "1:1",
- "url" => $result["visualWebImageSnippet"]
- ];
- }else{
- $thumb = [
- "ratio" => null,
- "url" => null
- ];
- }
-
- $sublinks = [];
-
- if(isset($result["megaSiteSubLinks"]["mssl"])){
-
- foreach($result["megaSiteSubLinks"]["mssl"] as $sublink){
-
- $sublinks[] = [
- "title" =>
- $this->titledots(
- $this->fuckhtml
- ->getTextContent(
- $sublink["title"]
- )
- ),
- "description" =>
- $this->titledots(
- html_entity_decode(
- $this->fuckhtml
- ->getTextContent(
- $sublink["description"]
- )
- )
- ),
- "url" => $sublink["url"],
- "date" => null
- ];
- }
- }
-
- $out["web"][] = [
- "title" =>
- $this->titledots(
- $this->fuckhtml
- ->getTextContent(
- $result["title"]
- )
- ),
- "description" =>
- $this->titledots(
- html_entity_decode(
- $this->fuckhtml
- ->getTextContent(
- $result["description"]
- )
- )
- ),
- "url" => $result["url"],
- "date" => isset($result["bylinedate"]) ? (int)$result["bylinedate"] : null,
- "type" => "web",
- "thumb" => $thumb,
- "sublink" => $sublinks,
- "table" => []
- ];
-
- if(isset($result["anotherSuggest"]["exploreQueries"])){
-
- foreach($result["anotherSuggest"]["exploreQueries"] as $query){
-
- $out["related"][] = $query["query"];
- }
- }
- break;
- }
- }
-
- //
- // Extract extras from "shortcuts"
- //
- foreach($json["props"]["pageProps"]["initialProps"]["pageData"]["shortcuts"] as $shortcut_wrap){
-
- foreach($shortcut_wrap as $shortcut){
-
- switch($shortcut["type"]){
- //
- // Scrape videos
- //
- case "GoogleVideoUniversalShortcut":
- foreach($shortcut["videos"] as $video){
-
- if(isset($video["thumbnailUrl"])){
-
- $thumb = [
- "ratio" => "16:9",
- "url" => $video["thumbnailUrl"]
- ];
- }else{
-
- $thumb = [
- "ratio" => null,
- "url" => null
- ];
- }
-
- if(isset($video["publishedDate"])){
-
- $date = strtotime($video["publishedDate"]);
-
- if($date === false){
-
- $date = null;
- }
- }else{
-
- $date = null;
- }
-
- $out["video"][] = [
- "title" =>
- $this->titledots(
- $this->fuckhtml
- ->getTextContent(
- $video["title"]
- )
- ),
- "description" => null,
- "date" => $date,
- "duration" =>
- isset($video["duration"]) ?
- $this->hms2int($video["duration"]) : null,
- "views" => null,
- "thumb" => $thumb,
- "url" => $video["url"]
- ];
- }
- break;
-
- //
- // Scrape images
- //
- case "ImageShortcut":
- foreach($shortcut["images"] as $image_cat){
-
- foreach($image_cat as $image){
-
- $ratio =
- $this->yahooratio(
- (int)$image["originalImageWidth"],
- (int)$image["originalImageHeight"]
- );
-
- $out["image"][] = [
- "title" =>
- $this->titledots(
- $this->fuckhtml
- ->getTextContent(
- $image["title"]
- )
- ),
- "source" => [
- [
- "url" => $image["originalImageUrl"],
- "width" => (int)$image["originalImageWidth"],
- "height" => (int)$image["originalImageHeight"]
- ],
- [
- "url" => $image["thumbnailUrl"],
- "width" => $ratio[0],
- "height" => $ratio[1]
- ]
- ],
- "url" => $image["referrerUrl"]
- ];
- }
- }
- break;
-
- case "GoogleRelatedQuestionsShortcut":
- foreach($shortcut["relatedQuestions"] as $question){
-
- //
- // Scrape answers, present them as search results
- //
- if(isset($question["result"]["thumbnailsInfo"][0]["thumbnailUrl"])){
-
- $thumb = [
- "ratio" => "16:9",
- "url" => $question["result"]["thumbnailsInfo"][0]["thumbnailUrl"]
- ];
- }else{
-
- $thumb = [
- "ratio" => null,
- "url" => null
- ];
- }
-
- $out["web"][] = [
- "title" =>
- $this->titledots(
- $this->fuckhtml
- ->getTextContent(
- $question["result"]["title"]
- )
- ),
- "description" =>
- $this->titledots(
- html_entity_decode(
- $this->fuckhtml
- ->getTextContent(
- $question["result"]["answer"]
- )
- )
- ),
- "url" => $question["result"]["url"],
- "date" => null,
- "type" => "web",
- "thumb" => $thumb,
- "sublink" => [],
- "table" => []
- ];
- }
- break;
-
- case "NewsShortcut":
- foreach($shortcut["results"] as $news){
-
- if(isset($news["imageUrl"])){
-
- $thumb = [
- "ratio" => "16:9",
- "url" => $news["imageUrl"]
- ];
- }else{
-
- $thumb = [
- "ratio" => null,
- "url" => null
- ];
- }
-
- $out["news"][] = [
- "title" => $news["headLine"],
- "description" =>
- $this->fuckhtml
- ->getTextContent(
- $news["text"]
- ),
- "date" => (int)$news["publishTime"],
- "thumb" => $thumb,
- "url" => $news["newsLink"]
- ];
- }
- break;
- }
- }
- }
-
- // get next page
- if(isset($json["props"]["pageProps"]["initialProps"]["pageData"]["pager"]["nextPage"])){
-
- $out["npt"] =
- $this->backend->store(
- $json["props"]["pageProps"]["initialProps"]["pageData"]["pager"]["nextPage"],
- "web",
- $proxy
- );
- }
-
- $out["related"] = array_unique($out["related"]);
-
- return $out;
- }
-
- public function image($get){
-
- $out = [
- "status" => "ok",
- "npt" => null,
- "image" => []
- ];
-
- if($get["npt"]){
-
- // parse JSON endpoint
- // https://search.yahoo.co.jp/image/api/search
- // ?p=minecraft
- // &ei=UTF-8
- // &n=20
- // &b=41
- // &vm=i
- // &cr=AiXVLGkAPLAUueqkG0dtUP6lo_3suz4Qsrv2QjabeXt4sk1wT8irS3LLvkSPRm-u7T1wvkE1ucQvhzYuB2QtDkjswUogRjoQVx_p73BaN3P1klQUFsnIPdgAttusXE0ii0pOcYCT
- // &se=0
- // &ue=0
- [$params, $proxy] = $this->backend->get($get["npt"], "images");
-
- $params = json_decode($params, true);
-
- // increment
- $params["params"]["b"] += 20;
-
- try{
- $json = $this->get(
- $proxy,
- "https://search.yahoo.co.jp/image/api/search",
- $params["params"],
- false,
- true,
- $params["cookies"]
- );
- }catch(Exception $error){
-
- throw new Exception("Failed to fetch JSON");
- }
-
- $json = json_decode($json, true);
-
- if($json === null){
-
- throw new Exception("Failed to decode JSON");
- }
-
- if(isset($json["Error"]["Message"])){
-
- throw new Exception("API returned an error: {$json["Error"]["Message"]}");
- }
-
- foreach($json["algos"] as $image){
-
- $out["image"][] = [
- "title" => $this->titledots($image["title"]),
- "source" => [
- [
- "url" => $image["original"]["url"],
- "width" => (int)$image["original"]["width"],
- "height" => (int)$image["original"]["height"]
- ],
- [
- "url" => $image["thumbnail"]["url"],
- "width" => (int)$image["thumbnail"]["width"],
- "height" => (int)$image["thumbnail"]["height"]
- ]
- ],
- "url" => $image["refererUrl"]
- ];
- }
-
- // detect next page
- if($json["algoAttribute"]["resultsIsLast"] === false){
-
- $out["npt"] =
- $this->backend->store(
- json_encode($params),
- "images",
- $proxy
- );
- }
-
- }else{
-
- // parse initial page
- $params = [
- "p" => $get["s"],
- "ei" => "UTF-8"
- ];
-
- $proxy = $this->backend->get_ip();
- try{
- $html = $this->get(
- $proxy,
- "https://search.yahoo.co.jp/image/search",
- $params,
- true
- );
- }catch(Exception $error){
-
- throw new Exception("Failed to fetch search page");
- }
- //$html = file_get_contents("scraper/yahoo_japan.html");
-
- $json_object =
- explode(
- '<script id="__NEXT_DATA__" type="application/json">',
- $html["body"]
- );
-
- if(count($json_object) !== 2){
-
- throw new Exception("Failed to find JSON script");
- }
-
- $json =
- json_decode(
- $this->fuckhtml
- ->extract_json(
- $json_object[1]
- ),
- true
- );
-
- if($json === null){
-
- throw new Exception("Failed to decode JSON");
- }
-
- if(!isset($json["props"]["initialProps"]["pageProps"]["algos"])){
-
- throw new Exception("Failed to access algos object");
- }
-
- foreach($json["props"]["initialProps"]["pageProps"]["algos"] as $image){
-
- $out["image"][] = [
- "title" => $this->titledots($image["title"]),
- "source" => [
- [
- "url" => $image["original"]["url"],
- "width" => (int)$image["original"]["width"],
- "height" => (int)$image["original"]["height"]
- ],
- [
- "url" => $image["thumbnail"]["url"],
- "width" => (int)$image["thumbnail"]["width"],
- "height" => (int)$image["thumbnail"]["height"]
- ]
- ],
- "url" => $image["refererUrl"]
- ];
- }
-
- // get next page
- if(
- $json["props"]["initialProps"]["pageProps"]["algoAttribute"]["resultsIsLast"] === false &&
- isset($json["props"]["initialProps"]["pageProps"]["crumb"]["crumbValue"])
- ){
-
- $out["npt"] =
- $this->backend->store(
- json_encode([
- "params" => [
- "p" => $get["s"],
- "ei" => "UTF-8",
- "n" => 20, // number of results
- "b" => 1, // increment (+20 on every page)
- "vm" => "i",
- "cr" => $json["props"]["initialProps"]["pageProps"]["crumb"]["crumbValue"],
- "se" => 0,
- "ue" => 0
- ],
- "cookies" => $html["cookies"]
- ]),
- "images",
- $proxy
- );
- }
- }
-
- return $out;
- }
-
- public function video($get){
-
- $out = [
- "status" => "ok",
- "npt" => null,
- "video" => [],
- "author" => [],
- "livestream" => [],
- "playlist" => [],
- "reel" => []
- ];
-
- if($get["npt"]){
-
- // parse JSON endpoint
- // https://search.yahoo.co.jp/image/api/search
- // ?p=minecraft
- // &ei=UTF-8
- // &n=20
- // &b=41
- // &vm=i
- // &cr=AiXVLGkAPLAUueqkG0dtUP6lo_3suz4Qsrv2QjabeXt4sk1wT8irS3LLvkSPRm-u7T1wvkE1ucQvhzYuB2QtDkjswUogRjoQVx_p73BaN3P1klQUFsnIPdgAttusXE0ii0pOcYCT
- // &se=0
- // &ue=0
- [$params, $proxy] = $this->backend->get($get["npt"], "images");
-
- $params = json_decode($params, true);
-
- // increment
- $params["params"]["b"] += 20;
-
- try{
- $json = $this->get(
- $proxy,
- "https://search.yahoo.co.jp/video/api/search",
- $params["params"],
- false,
- true,
- $params["cookies"]
- );
- }catch(Exception $error){
-
- throw new Exception("Failed to fetch JSON");
- }
-
- $json = json_decode($json, true);
-
- if($json === null){
-
- throw new Exception("Failed to decode JSON");
- }
-
- if(isset($json["Error"]["Message"])){
-
- throw new Exception("API returned an error: {$json["Error"]["Message"]}");
- }
-
- foreach($json["algos"] as $video){
-
- if(isset($video["uploadDate"])){
-
- $date = strtotime($video["uploadDate"]);
- if($date === false){
-
- $date = null;
- }
-
- }else{
-
- $date = null;
- }
-
- if(isset($video["thumbnail"]["url"])){
-
- $thumb = [
- "ratio" => "16:9",
- "url" => $video["thumbnail"]["url"]
- ];
- }else{
-
- $thumb = [
- "ratio" => null,
- "url" => null
- ];
- }
-
- $out["video"][] = [
- "title" => $this->titledots($video["title"]),
- "description" =>
- $this->titledots(
- html_entity_decode(
- $this->fuckhtml
- ->getTextContent(
- $video["summary"]
- )
- )
- ),
- "author" => [
- "name" =>
- (
- isset($video["uploader"]) &&
- $video["uploader"] != ""
- ) ?
- $video["uploader"] : null,
- "url" => null,
- "avatar" => null
- ],
- "date" => $date,
- "duration" =>
- (
- isset($video["duration"]) &&
- $video["duration"] != ""
- ) ?
- $this->hms2int($video["duration"]) : null,
- "views" => null,
- "thumb" => $thumb,
- "url" => $video["refererUrl"]
- ];
- }
-
- // detect next page
- if($json["algoAttribute"]["isLast"] === false){
-
- $out["npt"] =
- $this->backend->store(
- json_encode($params),
- "images",
- $proxy
- );
- }
-
- }else{
-
- // parse initial page
- $params = [
- "p" => $get["s"],
- "ei" => "UTF-8"
- ];
-
- $proxy = $this->backend->get_ip();
- try{
- $html = $this->get(
- $proxy,
- "https://search.yahoo.co.jp/video/search",
- $params,
- true
- );
- }catch(Exception $error){
-
- throw new Exception("Failed to fetch search page");
- }
- //$html = file_get_contents("scraper/yahoo_japan.html");
-
- $json_object =
- explode(
- '<script id="__NEXT_DATA__" type="application/json">',
- $html["body"]
- );
-
- if(count($json_object) !== 2){
-
- throw new Exception("Failed to find JSON script");
- }
-
- $json =
- json_decode(
- $this->fuckhtml
- ->extract_json(
- $json_object[1]
- ),
- true
- );
-
- if($json === null){
-
- throw new Exception("Failed to decode JSON");
- }
-
- if(!isset($json["props"]["initialProps"]["pageProps"]["algos"])){
-
- throw new Exception("Failed to access algos object");
- }
-
- foreach($json["props"]["initialProps"]["pageProps"]["algos"] as $video){
-
- if(isset($video["uploadDate"])){
-
- $date = strtotime($video["uploadDate"]);
- if($date === false){
-
- $date = null;
- }
-
- }else{
-
- $date = null;
- }
-
- if(isset($video["thumbnail"]["url"])){
-
- $thumb = [
- "ratio" => "16:9",
- "url" => $video["thumbnail"]["url"]
- ];
- }else{
-
- $thumb = [
- "ratio" => null,
- "url" => null
- ];
- }
-
- $out["video"][] = [
- "title" => $this->titledots($video["title"]),
- "description" =>
- $this->titledots(
- html_entity_decode(
- $this->fuckhtml
- ->getTextContent(
- $video["summary"]
- )
- )
- ),
- "author" => [
- "name" =>
- (
- isset($video["uploader"]) &&
- $video["uploader"] != ""
- ) ?
- $video["uploader"] : null,
- "url" => null,
- "avatar" => null
- ],
- "date" => $date,
- "duration" =>
- (
- isset($video["duration"]) &&
- $video["duration"] != ""
- ) ?
- $this->hms2int($video["duration"]) : null,
- "views" => null,
- "thumb" => $thumb,
- "url" => $video["refererUrl"]
- ];
- }
-
- // get next page
- if(
- $json["props"]["initialProps"]["pageProps"]["algoAttribute"]["isLast"] === false &&
- isset($json["props"]["initialProps"]["pageProps"]["crumb"]["crumbValue"])
- ){
-
- $out["npt"] =
- $this->backend->store(
- json_encode([
- "params" => [
- "n" => 20, // number of results
- "b" => 1, // increment (+20)
- "vm" => "i",
- "cr" => $json["props"]["initialProps"]["pageProps"]["crumb"]["crumbValue"],
- "p" => $get["s"],
- "pd" => "",
- "dr" => "",
- "hq" => "",
- "st" => "",
- "qrw" => "",
- "ei" => "UTF-8",
- "ue" => "0",
- "se" => "0"
- ],
- "cookies" => $html["cookies"]
- ]),
- "images",
- $proxy
- );
- }
- }
-
- return $out;
- }
-
- public function news($get){
-
- if($get["npt"]){
-
- [$params, $proxy] = $this->backend->get($get["npt"], "news");
- $params = json_decode($params, true);
-
- }else{
-
- $search = $get["s"];
- if(strlen($search) === 0){
-
- throw new Exception("Search term is empty!");
- }
-
- $proxy = $this->backend->get_ip();
-
- $params = [
- "p" => $get["s"],
- "ei" => "UTF-8"
- ];
- }
-
- try{
- $html = $this->get(
- $proxy,
- "https://chiebukuro.yahoo.co.jp/search",
- $params
- );
- }catch(Exception $error){
-
- throw new Exception("Failed to fetch search page");
- }
- //$html = file_get_contents("scraper/yahoo_japan.html");
-
- $out = [
- "status" => "ok",
- "npt" => null,
- "news" => []
- ];
-
- $json_object =
- explode(
- 'window.PROPS = ',
- $html
- );
-
- if(count($json_object) !== 2){
-
- throw new Exception("Failed to find JSON script");
- }
-
- $json =
- json_decode(
- $this->fuckhtml
- ->extract_json(
- $json_object[1]
- ),
- true
- );
-
- if($json === null){
-
- throw new Exception("Failed to decode JSON");
- }
-
- if(!isset($json["listSearchResults"]["listContents"])){
-
- throw new Exception("Yahoo! did not return a listContents object");
- }
-
- foreach($json["listSearchResults"]["listContents"] as $news){
-
- $date = strtotime($news["datePosted"]);
-
- if($date === false){
-
- $date = null;
- }
-
- $thumb = [
- "ratio" => null,
- "url" => null
- ];
-
- $out["news"][] = [
- "title" =>
- $this->titledots(
- $this->fuckhtml
- ->getTextContent(
- $news["heading"]
- )
- ),
- "author" => null,
- "description" =>
- $this->titledots(
- $this->fuckhtml
- ->getTextContent(
- $news["summary"]
- )
- ),
- "date" => $date,
- "thumb" => $thumb,
- "url" => $news["url"]
- ];
- }
-
- // get next page
- if($json["pagination"]["currentPage"] != $json["pagination"]["totalNumberOfPages"]){
-
- if(!isset($params["b"])){
-
- $params["b"] = 1;
- }
-
- $params["b"] += 10;
-
- $out["npt"] =
- $this->backend->store(
- json_encode($params),
- "news",
- $proxy
- );
- }
-
- return $out;
- }
-
- private function hms2int($time){
-
- $parts = explode(":", $time, 3);
- $time = 0;
-
- if(count($parts) === 3){
-
- // hours
- $time = $time + ((int)$parts[0] * 3600);
- array_shift($parts);
- }
-
- if(count($parts) === 2){
-
- // minutes
- $time = $time + ((int)$parts[0] * 60);
- array_shift($parts);
- }
-
- // seconds
- $time = $time + (int)$parts[0];
-
- return $time;
- }
-
- private function titledots($title){
-
- $substr = substr($title, -3);
-
- if(
- $substr == "..." ||
- $substr == "…"
- ){
-
- return trim(substr($title, 0, -3));
- }
-
- return trim($title);
- }
-
- private function yahooratio($width, $height){
-
- $ratio = [
- 144 / $width,
- 256 / $height
- ];
-
- if($ratio[0] < $ratio[1]){
-
- $ratio = $ratio[0];
- }else{
-
- $ratio = $ratio[1];
- }
-
- return [
- floor($width * $ratio),
- floor($height * $ratio)
- ];
- }
- }
|