yahoo_japan.php 23 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162
  1. <?php
  2. class yahoo_japan{
  3. public function __construct(){
  4. include "lib/backend.php";
  5. $this->backend = new backend("yahoo_japan");
  6. include "lib/fuckhtml.php";
  7. $this->fuckhtml = new fuckhtml();
  8. }
  9. public function getfilters($page){
  10. return [];
  11. }
  12. private function get($proxy, $url, $get = [], $return_cookies = false, $is_xhr = false, $cookie = null){
  13. $curlproc = curl_init();
  14. if($get !== []){
  15. $get = http_build_query($get);
  16. $url .= "?" . $get;
  17. }
  18. curl_setopt($curlproc, CURLOPT_URL, $url);
  19. // http2 bypass
  20. curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
  21. curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
  22. if($cookie !== null){
  23. $c = [];
  24. foreach($cookie as $name => $value){
  25. $c[] = "{$name}=$value";
  26. }
  27. $cookie = implode("; ", $c);
  28. }
  29. if($is_xhr){
  30. curl_setopt($curlproc, CURLOPT_HTTPHEADER,
  31. ["User-Agent: " . config::USER_AGENT,
  32. "Accept: application/json, text/plain, */*",
  33. "Accept-Language: en-US,en;q=0.5",
  34. "Accept-Encoding: gzip",
  35. "Referer: https://search.yahoo.co.jp/",
  36. "DNT: 1",
  37. "Sec-GPC: 1",
  38. "Connection: keep-alive",
  39. "Cookie: " . $cookie,
  40. "Sec-Fetch-Dest: empty",
  41. "Sec-Fetch-Mode: cors",
  42. "Sec-Fetch-Site: same-origin",
  43. "TE: trailers"]
  44. );
  45. }else{
  46. curl_setopt($curlproc, CURLOPT_HTTPHEADER,
  47. ["User-Agent: " . config::USER_AGENT,
  48. "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
  49. "Accept-Language: en-US,en;q=0.5",
  50. "Accept-Encoding: gzip",
  51. "DNT: 1",
  52. "Sec-GPC: 1",
  53. "Connection: keep-alive",
  54. "Upgrade-Insecure-Requests: 1",
  55. "Sec-Fetch-Dest: document",
  56. "Sec-Fetch-Mode: navigate",
  57. "Sec-Fetch-Site: same-origin",
  58. "Sec-Fetch-User: ?1",
  59. "Priority: u=0, i",
  60. "TE: trailers"]
  61. );
  62. }
  63. curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
  64. curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
  65. curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
  66. curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
  67. curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
  68. $this->backend->assign_proxy($curlproc, $proxy);
  69. if($return_cookies){
  70. // extract cookies
  71. $cookies_tmp = [];
  72. curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies_tmp){
  73. $length = strlen($header);
  74. $header = explode(":", $header, 2);
  75. if(trim(strtolower($header[0])) == "set-cookie"){
  76. $cookie_tmp = explode("=", trim($header[1]), 2);
  77. $cookies_tmp[trim($cookie_tmp[0])] =
  78. explode(";", $cookie_tmp[1], 2)[0];
  79. }
  80. return $length;
  81. });
  82. }
  83. $data = curl_exec($curlproc);
  84. if(curl_errno($curlproc)){
  85. throw new Exception(curl_error($curlproc));
  86. }
  87. curl_close($curlproc);
  88. if($return_cookies){
  89. return [
  90. "cookies" => $cookies_tmp,
  91. "body" => $data
  92. ];
  93. }
  94. return $data;
  95. }
  96. public function web($get){
  97. if($get["npt"]){
  98. [$url, $proxy] = $this->backend->get($get["npt"], "web");
  99. $params = [];
  100. }else{
  101. $search = $get["s"];
  102. if(strlen($search) === 0){
  103. throw new Exception("Search term is empty!");
  104. }
  105. $proxy = $this->backend->get_ip();
  106. $url = "https://search.yahoo.co.jp/search";
  107. $params = [
  108. "p" => $get["s"]
  109. ];
  110. }
  111. try{
  112. $html = $this->get(
  113. $proxy,
  114. $url,
  115. $params
  116. );
  117. }catch(Exception $error){
  118. throw new Exception("Failed to fetch search page");
  119. }
  120. //$html = file_get_contents("scraper/yahoo_japan.html");
  121. $out = [
  122. "status" => "ok",
  123. "spelling" => [
  124. "type" => "no_correction",
  125. "using" => null,
  126. "correction" => null
  127. ],
  128. "npt" => null,
  129. "answer" => [],
  130. "web" => [],
  131. "image" => [],
  132. "video" => [],
  133. "news" => [],
  134. "related" => []
  135. ];
  136. $json_object =
  137. explode(
  138. '<script id="__NEXT_DATA__" type="application/json">',
  139. $html
  140. );
  141. if(count($json_object) !== 2){
  142. throw new Exception("Failed to find JSON script");
  143. }
  144. $json =
  145. json_decode(
  146. $this->fuckhtml
  147. ->extract_json(
  148. $json_object[1]
  149. ),
  150. true
  151. );
  152. if($json === null){
  153. throw new Exception("Failed to decode JSON");
  154. }
  155. //print_r($json);
  156. //
  157. // Extract mainline search results
  158. //
  159. if(!isset($json["props"]["pageProps"]["initialProps"]["pageData"]["algos"])){
  160. throw new Exception("Failed to access algos object");
  161. }
  162. foreach($json["props"]["pageProps"]["initialProps"]["pageData"]["algos"] as $result){
  163. switch($result["type"]){
  164. case "Algo":
  165. if(isset($result["visualWebImageGallery"]["imageThumbs"][0]["source"])){
  166. $thumb = [
  167. "ratio" => "1:1",
  168. "url" => $result["visualWebImageGallery"]["imageThumbs"][0]["source"]
  169. ];
  170. }elseif(isset($result["visualWebImageSnippet"])){
  171. $thumb = [
  172. "ratio" => "1:1",
  173. "url" => $result["visualWebImageSnippet"]
  174. ];
  175. }else{
  176. $thumb = [
  177. "ratio" => null,
  178. "url" => null
  179. ];
  180. }
  181. $sublinks = [];
  182. if(isset($result["megaSiteSubLinks"]["mssl"])){
  183. foreach($result["megaSiteSubLinks"]["mssl"] as $sublink){
  184. $sublinks[] = [
  185. "title" =>
  186. $this->titledots(
  187. $this->fuckhtml
  188. ->getTextContent(
  189. $sublink["title"]
  190. )
  191. ),
  192. "description" =>
  193. $this->titledots(
  194. html_entity_decode(
  195. $this->fuckhtml
  196. ->getTextContent(
  197. $sublink["description"]
  198. )
  199. )
  200. ),
  201. "url" => $sublink["url"],
  202. "date" => null
  203. ];
  204. }
  205. }
  206. $out["web"][] = [
  207. "title" =>
  208. $this->titledots(
  209. $this->fuckhtml
  210. ->getTextContent(
  211. $result["title"]
  212. )
  213. ),
  214. "description" =>
  215. $this->titledots(
  216. html_entity_decode(
  217. $this->fuckhtml
  218. ->getTextContent(
  219. $result["description"]
  220. )
  221. )
  222. ),
  223. "url" => $result["url"],
  224. "date" => isset($result["bylinedate"]) ? (int)$result["bylinedate"] : null,
  225. "type" => "web",
  226. "thumb" => $thumb,
  227. "sublink" => $sublinks,
  228. "table" => []
  229. ];
  230. if(isset($result["anotherSuggest"]["exploreQueries"])){
  231. foreach($result["anotherSuggest"]["exploreQueries"] as $query){
  232. $out["related"][] = $query["query"];
  233. }
  234. }
  235. break;
  236. }
  237. }
  238. //
  239. // Extract extras from "shortcuts"
  240. //
  241. foreach($json["props"]["pageProps"]["initialProps"]["pageData"]["shortcuts"] as $shortcut_wrap){
  242. foreach($shortcut_wrap as $shortcut){
  243. switch($shortcut["type"]){
  244. //
  245. // Scrape videos
  246. //
  247. case "GoogleVideoUniversalShortcut":
  248. foreach($shortcut["videos"] as $video){
  249. if(isset($video["thumbnailUrl"])){
  250. $thumb = [
  251. "ratio" => "16:9",
  252. "url" => $video["thumbnailUrl"]
  253. ];
  254. }else{
  255. $thumb = [
  256. "ratio" => null,
  257. "url" => null
  258. ];
  259. }
  260. if(isset($video["publishedDate"])){
  261. $date = strtotime($video["publishedDate"]);
  262. if($date === false){
  263. $date = null;
  264. }
  265. }else{
  266. $date = null;
  267. }
  268. $out["video"][] = [
  269. "title" =>
  270. $this->titledots(
  271. $this->fuckhtml
  272. ->getTextContent(
  273. $video["title"]
  274. )
  275. ),
  276. "description" => null,
  277. "date" => $date,
  278. "duration" =>
  279. isset($video["duration"]) ?
  280. $this->hms2int($video["duration"]) : null,
  281. "views" => null,
  282. "thumb" => $thumb,
  283. "url" => $video["url"]
  284. ];
  285. }
  286. break;
  287. //
  288. // Scrape images
  289. //
  290. case "ImageShortcut":
  291. foreach($shortcut["images"] as $image_cat){
  292. foreach($image_cat as $image){
  293. $ratio =
  294. $this->yahooratio(
  295. (int)$image["originalImageWidth"],
  296. (int)$image["originalImageHeight"]
  297. );
  298. $out["image"][] = [
  299. "title" =>
  300. $this->titledots(
  301. $this->fuckhtml
  302. ->getTextContent(
  303. $image["title"]
  304. )
  305. ),
  306. "source" => [
  307. [
  308. "url" => $image["originalImageUrl"],
  309. "width" => (int)$image["originalImageWidth"],
  310. "height" => (int)$image["originalImageHeight"]
  311. ],
  312. [
  313. "url" => $image["thumbnailUrl"],
  314. "width" => $ratio[0],
  315. "height" => $ratio[1]
  316. ]
  317. ],
  318. "url" => $image["referrerUrl"]
  319. ];
  320. }
  321. }
  322. break;
  323. case "GoogleRelatedQuestionsShortcut":
  324. foreach($shortcut["relatedQuestions"] as $question){
  325. //
  326. // Scrape answers, present them as search results
  327. //
  328. if(isset($question["result"]["thumbnailsInfo"][0]["thumbnailUrl"])){
  329. $thumb = [
  330. "ratio" => "16:9",
  331. "url" => $question["result"]["thumbnailsInfo"][0]["thumbnailUrl"]
  332. ];
  333. }else{
  334. $thumb = [
  335. "ratio" => null,
  336. "url" => null
  337. ];
  338. }
  339. $out["web"][] = [
  340. "title" =>
  341. $this->titledots(
  342. $this->fuckhtml
  343. ->getTextContent(
  344. $question["result"]["title"]
  345. )
  346. ),
  347. "description" =>
  348. $this->titledots(
  349. html_entity_decode(
  350. $this->fuckhtml
  351. ->getTextContent(
  352. $question["result"]["answer"]
  353. )
  354. )
  355. ),
  356. "url" => $question["result"]["url"],
  357. "date" => null,
  358. "type" => "web",
  359. "thumb" => $thumb,
  360. "sublink" => [],
  361. "table" => []
  362. ];
  363. }
  364. break;
  365. case "NewsShortcut":
  366. foreach($shortcut["results"] as $news){
  367. if(isset($news["imageUrl"])){
  368. $thumb = [
  369. "ratio" => "16:9",
  370. "url" => $news["imageUrl"]
  371. ];
  372. }else{
  373. $thumb = [
  374. "ratio" => null,
  375. "url" => null
  376. ];
  377. }
  378. $out["news"][] = [
  379. "title" => $news["headLine"],
  380. "description" =>
  381. $this->fuckhtml
  382. ->getTextContent(
  383. $news["text"]
  384. ),
  385. "date" => (int)$news["publishTime"],
  386. "thumb" => $thumb,
  387. "url" => $news["newsLink"]
  388. ];
  389. }
  390. break;
  391. }
  392. }
  393. }
  394. // get next page
  395. if(isset($json["props"]["pageProps"]["initialProps"]["pageData"]["pager"]["nextPage"])){
  396. $out["npt"] =
  397. $this->backend->store(
  398. $json["props"]["pageProps"]["initialProps"]["pageData"]["pager"]["nextPage"],
  399. "web",
  400. $proxy
  401. );
  402. }
  403. $out["related"] = array_unique($out["related"]);
  404. return $out;
  405. }
  406. public function image($get){
  407. $out = [
  408. "status" => "ok",
  409. "npt" => null,
  410. "image" => []
  411. ];
  412. if($get["npt"]){
  413. // parse JSON endpoint
  414. // https://search.yahoo.co.jp/image/api/search
  415. // ?p=minecraft
  416. // &ei=UTF-8
  417. // &n=20
  418. // &b=41
  419. // &vm=i
  420. // &cr=AiXVLGkAPLAUueqkG0dtUP6lo_3suz4Qsrv2QjabeXt4sk1wT8irS3LLvkSPRm-u7T1wvkE1ucQvhzYuB2QtDkjswUogRjoQVx_p73BaN3P1klQUFsnIPdgAttusXE0ii0pOcYCT
  421. // &se=0
  422. // &ue=0
  423. [$params, $proxy] = $this->backend->get($get["npt"], "images");
  424. $params = json_decode($params, true);
  425. // increment
  426. $params["params"]["b"] += 20;
  427. try{
  428. $json = $this->get(
  429. $proxy,
  430. "https://search.yahoo.co.jp/image/api/search",
  431. $params["params"],
  432. false,
  433. true,
  434. $params["cookies"]
  435. );
  436. }catch(Exception $error){
  437. throw new Exception("Failed to fetch JSON");
  438. }
  439. $json = json_decode($json, true);
  440. if($json === null){
  441. throw new Exception("Failed to decode JSON");
  442. }
  443. if(isset($json["Error"]["Message"])){
  444. throw new Exception("API returned an error: {$json["Error"]["Message"]}");
  445. }
  446. foreach($json["algos"] as $image){
  447. $out["image"][] = [
  448. "title" => $this->titledots($image["title"]),
  449. "source" => [
  450. [
  451. "url" => $image["original"]["url"],
  452. "width" => (int)$image["original"]["width"],
  453. "height" => (int)$image["original"]["height"]
  454. ],
  455. [
  456. "url" => $image["thumbnail"]["url"],
  457. "width" => (int)$image["thumbnail"]["width"],
  458. "height" => (int)$image["thumbnail"]["height"]
  459. ]
  460. ],
  461. "url" => $image["refererUrl"]
  462. ];
  463. }
  464. // detect next page
  465. if($json["algoAttribute"]["resultsIsLast"] === false){
  466. $out["npt"] =
  467. $this->backend->store(
  468. json_encode($params),
  469. "images",
  470. $proxy
  471. );
  472. }
  473. }else{
  474. // parse initial page
  475. $params = [
  476. "p" => $get["s"],
  477. "ei" => "UTF-8"
  478. ];
  479. $proxy = $this->backend->get_ip();
  480. try{
  481. $html = $this->get(
  482. $proxy,
  483. "https://search.yahoo.co.jp/image/search",
  484. $params,
  485. true
  486. );
  487. }catch(Exception $error){
  488. throw new Exception("Failed to fetch search page");
  489. }
  490. //$html = file_get_contents("scraper/yahoo_japan.html");
  491. $json_object =
  492. explode(
  493. '<script id="__NEXT_DATA__" type="application/json">',
  494. $html["body"]
  495. );
  496. if(count($json_object) !== 2){
  497. throw new Exception("Failed to find JSON script");
  498. }
  499. $json =
  500. json_decode(
  501. $this->fuckhtml
  502. ->extract_json(
  503. $json_object[1]
  504. ),
  505. true
  506. );
  507. if($json === null){
  508. throw new Exception("Failed to decode JSON");
  509. }
  510. if(!isset($json["props"]["initialProps"]["pageProps"]["algos"])){
  511. throw new Exception("Failed to access algos object");
  512. }
  513. foreach($json["props"]["initialProps"]["pageProps"]["algos"] as $image){
  514. $out["image"][] = [
  515. "title" => $this->titledots($image["title"]),
  516. "source" => [
  517. [
  518. "url" => $image["original"]["url"],
  519. "width" => (int)$image["original"]["width"],
  520. "height" => (int)$image["original"]["height"]
  521. ],
  522. [
  523. "url" => $image["thumbnail"]["url"],
  524. "width" => (int)$image["thumbnail"]["width"],
  525. "height" => (int)$image["thumbnail"]["height"]
  526. ]
  527. ],
  528. "url" => $image["refererUrl"]
  529. ];
  530. }
  531. // get next page
  532. if(
  533. $json["props"]["initialProps"]["pageProps"]["algoAttribute"]["resultsIsLast"] === false &&
  534. isset($json["props"]["initialProps"]["pageProps"]["crumb"]["crumbValue"])
  535. ){
  536. $out["npt"] =
  537. $this->backend->store(
  538. json_encode([
  539. "params" => [
  540. "p" => $get["s"],
  541. "ei" => "UTF-8",
  542. "n" => 20, // number of results
  543. "b" => 1, // increment (+20 on every page)
  544. "vm" => "i",
  545. "cr" => $json["props"]["initialProps"]["pageProps"]["crumb"]["crumbValue"],
  546. "se" => 0,
  547. "ue" => 0
  548. ],
  549. "cookies" => $html["cookies"]
  550. ]),
  551. "images",
  552. $proxy
  553. );
  554. }
  555. }
  556. return $out;
  557. }
  558. public function video($get){
  559. $out = [
  560. "status" => "ok",
  561. "npt" => null,
  562. "video" => [],
  563. "author" => [],
  564. "livestream" => [],
  565. "playlist" => [],
  566. "reel" => []
  567. ];
  568. if($get["npt"]){
  569. // parse JSON endpoint
  570. // https://search.yahoo.co.jp/image/api/search
  571. // ?p=minecraft
  572. // &ei=UTF-8
  573. // &n=20
  574. // &b=41
  575. // &vm=i
  576. // &cr=AiXVLGkAPLAUueqkG0dtUP6lo_3suz4Qsrv2QjabeXt4sk1wT8irS3LLvkSPRm-u7T1wvkE1ucQvhzYuB2QtDkjswUogRjoQVx_p73BaN3P1klQUFsnIPdgAttusXE0ii0pOcYCT
  577. // &se=0
  578. // &ue=0
  579. [$params, $proxy] = $this->backend->get($get["npt"], "images");
  580. $params = json_decode($params, true);
  581. // increment
  582. $params["params"]["b"] += 20;
  583. try{
  584. $json = $this->get(
  585. $proxy,
  586. "https://search.yahoo.co.jp/video/api/search",
  587. $params["params"],
  588. false,
  589. true,
  590. $params["cookies"]
  591. );
  592. }catch(Exception $error){
  593. throw new Exception("Failed to fetch JSON");
  594. }
  595. $json = json_decode($json, true);
  596. if($json === null){
  597. throw new Exception("Failed to decode JSON");
  598. }
  599. if(isset($json["Error"]["Message"])){
  600. throw new Exception("API returned an error: {$json["Error"]["Message"]}");
  601. }
  602. foreach($json["algos"] as $video){
  603. if(isset($video["uploadDate"])){
  604. $date = strtotime($video["uploadDate"]);
  605. if($date === false){
  606. $date = null;
  607. }
  608. }else{
  609. $date = null;
  610. }
  611. if(isset($video["thumbnail"]["url"])){
  612. $thumb = [
  613. "ratio" => "16:9",
  614. "url" => $video["thumbnail"]["url"]
  615. ];
  616. }else{
  617. $thumb = [
  618. "ratio" => null,
  619. "url" => null
  620. ];
  621. }
  622. $out["video"][] = [
  623. "title" => $this->titledots($video["title"]),
  624. "description" =>
  625. $this->titledots(
  626. html_entity_decode(
  627. $this->fuckhtml
  628. ->getTextContent(
  629. $video["summary"]
  630. )
  631. )
  632. ),
  633. "author" => [
  634. "name" =>
  635. (
  636. isset($video["uploader"]) &&
  637. $video["uploader"] != ""
  638. ) ?
  639. $video["uploader"] : null,
  640. "url" => null,
  641. "avatar" => null
  642. ],
  643. "date" => $date,
  644. "duration" =>
  645. (
  646. isset($video["duration"]) &&
  647. $video["duration"] != ""
  648. ) ?
  649. $this->hms2int($video["duration"]) : null,
  650. "views" => null,
  651. "thumb" => $thumb,
  652. "url" => $video["refererUrl"]
  653. ];
  654. }
  655. // detect next page
  656. if($json["algoAttribute"]["isLast"] === false){
  657. $out["npt"] =
  658. $this->backend->store(
  659. json_encode($params),
  660. "images",
  661. $proxy
  662. );
  663. }
  664. }else{
  665. // parse initial page
  666. $params = [
  667. "p" => $get["s"],
  668. "ei" => "UTF-8"
  669. ];
  670. $proxy = $this->backend->get_ip();
  671. try{
  672. $html = $this->get(
  673. $proxy,
  674. "https://search.yahoo.co.jp/video/search",
  675. $params,
  676. true
  677. );
  678. }catch(Exception $error){
  679. throw new Exception("Failed to fetch search page");
  680. }
  681. //$html = file_get_contents("scraper/yahoo_japan.html");
  682. $json_object =
  683. explode(
  684. '<script id="__NEXT_DATA__" type="application/json">',
  685. $html["body"]
  686. );
  687. if(count($json_object) !== 2){
  688. throw new Exception("Failed to find JSON script");
  689. }
  690. $json =
  691. json_decode(
  692. $this->fuckhtml
  693. ->extract_json(
  694. $json_object[1]
  695. ),
  696. true
  697. );
  698. if($json === null){
  699. throw new Exception("Failed to decode JSON");
  700. }
  701. if(!isset($json["props"]["initialProps"]["pageProps"]["algos"])){
  702. throw new Exception("Failed to access algos object");
  703. }
  704. foreach($json["props"]["initialProps"]["pageProps"]["algos"] as $video){
  705. if(isset($video["uploadDate"])){
  706. $date = strtotime($video["uploadDate"]);
  707. if($date === false){
  708. $date = null;
  709. }
  710. }else{
  711. $date = null;
  712. }
  713. if(isset($video["thumbnail"]["url"])){
  714. $thumb = [
  715. "ratio" => "16:9",
  716. "url" => $video["thumbnail"]["url"]
  717. ];
  718. }else{
  719. $thumb = [
  720. "ratio" => null,
  721. "url" => null
  722. ];
  723. }
  724. $out["video"][] = [
  725. "title" => $this->titledots($video["title"]),
  726. "description" =>
  727. $this->titledots(
  728. html_entity_decode(
  729. $this->fuckhtml
  730. ->getTextContent(
  731. $video["summary"]
  732. )
  733. )
  734. ),
  735. "author" => [
  736. "name" =>
  737. (
  738. isset($video["uploader"]) &&
  739. $video["uploader"] != ""
  740. ) ?
  741. $video["uploader"] : null,
  742. "url" => null,
  743. "avatar" => null
  744. ],
  745. "date" => $date,
  746. "duration" =>
  747. (
  748. isset($video["duration"]) &&
  749. $video["duration"] != ""
  750. ) ?
  751. $this->hms2int($video["duration"]) : null,
  752. "views" => null,
  753. "thumb" => $thumb,
  754. "url" => $video["refererUrl"]
  755. ];
  756. }
  757. // get next page
  758. if(
  759. $json["props"]["initialProps"]["pageProps"]["algoAttribute"]["isLast"] === false &&
  760. isset($json["props"]["initialProps"]["pageProps"]["crumb"]["crumbValue"])
  761. ){
  762. $out["npt"] =
  763. $this->backend->store(
  764. json_encode([
  765. "params" => [
  766. "n" => 20, // number of results
  767. "b" => 1, // increment (+20)
  768. "vm" => "i",
  769. "cr" => $json["props"]["initialProps"]["pageProps"]["crumb"]["crumbValue"],
  770. "p" => $get["s"],
  771. "pd" => "",
  772. "dr" => "",
  773. "hq" => "",
  774. "st" => "",
  775. "qrw" => "",
  776. "ei" => "UTF-8",
  777. "ue" => "0",
  778. "se" => "0"
  779. ],
  780. "cookies" => $html["cookies"]
  781. ]),
  782. "images",
  783. $proxy
  784. );
  785. }
  786. }
  787. return $out;
  788. }
  789. public function news($get){
  790. if($get["npt"]){
  791. [$params, $proxy] = $this->backend->get($get["npt"], "news");
  792. $params = json_decode($params, true);
  793. }else{
  794. $search = $get["s"];
  795. if(strlen($search) === 0){
  796. throw new Exception("Search term is empty!");
  797. }
  798. $proxy = $this->backend->get_ip();
  799. $params = [
  800. "p" => $get["s"],
  801. "ei" => "UTF-8"
  802. ];
  803. }
  804. try{
  805. $html = $this->get(
  806. $proxy,
  807. "https://chiebukuro.yahoo.co.jp/search",
  808. $params
  809. );
  810. }catch(Exception $error){
  811. throw new Exception("Failed to fetch search page");
  812. }
  813. //$html = file_get_contents("scraper/yahoo_japan.html");
  814. $out = [
  815. "status" => "ok",
  816. "npt" => null,
  817. "news" => []
  818. ];
  819. $json_object =
  820. explode(
  821. 'window.PROPS = ',
  822. $html
  823. );
  824. if(count($json_object) !== 2){
  825. throw new Exception("Failed to find JSON script");
  826. }
  827. $json =
  828. json_decode(
  829. $this->fuckhtml
  830. ->extract_json(
  831. $json_object[1]
  832. ),
  833. true
  834. );
  835. if($json === null){
  836. throw new Exception("Failed to decode JSON");
  837. }
  838. if(!isset($json["listSearchResults"]["listContents"])){
  839. throw new Exception("Yahoo! did not return a listContents object");
  840. }
  841. foreach($json["listSearchResults"]["listContents"] as $news){
  842. $date = strtotime($news["datePosted"]);
  843. if($date === false){
  844. $date = null;
  845. }
  846. $thumb = [
  847. "ratio" => null,
  848. "url" => null
  849. ];
  850. $out["news"][] = [
  851. "title" =>
  852. $this->titledots(
  853. $this->fuckhtml
  854. ->getTextContent(
  855. $news["heading"]
  856. )
  857. ),
  858. "author" => null,
  859. "description" =>
  860. $this->titledots(
  861. $this->fuckhtml
  862. ->getTextContent(
  863. $news["summary"]
  864. )
  865. ),
  866. "date" => $date,
  867. "thumb" => $thumb,
  868. "url" => $news["url"]
  869. ];
  870. }
  871. // get next page
  872. if($json["pagination"]["currentPage"] != $json["pagination"]["totalNumberOfPages"]){
  873. if(!isset($params["b"])){
  874. $params["b"] = 1;
  875. }
  876. $params["b"] += 10;
  877. $out["npt"] =
  878. $this->backend->store(
  879. json_encode($params),
  880. "news",
  881. $proxy
  882. );
  883. }
  884. return $out;
  885. }
  886. private function hms2int($time){
  887. $parts = explode(":", $time, 3);
  888. $time = 0;
  889. if(count($parts) === 3){
  890. // hours
  891. $time = $time + ((int)$parts[0] * 3600);
  892. array_shift($parts);
  893. }
  894. if(count($parts) === 2){
  895. // minutes
  896. $time = $time + ((int)$parts[0] * 60);
  897. array_shift($parts);
  898. }
  899. // seconds
  900. $time = $time + (int)$parts[0];
  901. return $time;
  902. }
  903. private function titledots($title){
  904. $substr = substr($title, -3);
  905. if(
  906. $substr == "..." ||
  907. $substr == "…"
  908. ){
  909. return trim(substr($title, 0, -3));
  910. }
  911. return trim($title);
  912. }
  913. private function yahooratio($width, $height){
  914. $ratio = [
  915. 144 / $width,
  916. 256 / $height
  917. ];
  918. if($ratio[0] < $ratio[1]){
  919. $ratio = $ratio[0];
  920. }else{
  921. $ratio = $ratio[1];
  922. }
  923. return [
  924. floor($width * $ratio),
  925. floor($height * $ratio)
  926. ];
  927. }
  928. }